From 49ec68e267b3e718d12bd435e3c06ddd2ed40d46 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 19 Aug 2026 18:07:08 -0700 Subject: [PATCH 1/9] refactor(compute): decouple gateway driver composition Move first-party composition and VM process ownership into openshell-gateway, leaving openshell-server backend-independent. Update packaging and build references with the new crate, simplify the compiled-driver boundary, and keep the driver-free gateway path buildable with bundled Z3 tooling. Signed-off-by: Drew Newberry --- .../skills/debug-openshell-cluster/SKILL.md | 5 +- .github/workflows/branch-checks.yml | 4 +- AGENTS.md | 1 + README.md | 2 +- architecture/build.md | 2 +- architecture/compute-runtimes.md | 42 +- crates/openshell-core/src/config.rs | 1291 +---------------- crates/openshell-core/src/driver_utils.rs | 13 +- ...lowlist.rs => dynamic_string_allowlist.rs} | 11 +- crates/openshell-core/src/error.rs | 4 +- crates/openshell-core/src/lib.rs | 14 +- crates/openshell-core/src/local_api_socket.rs | 81 ++ crates/openshell-core/src/settings.rs | 2 +- crates/openshell-core/src/telemetry.rs | 71 +- crates/openshell-driver-docker/src/lib.rs | 525 +------ crates/openshell-driver-docker/src/main.rs | 5 +- .../openshell-driver-kubernetes/src/config.rs | 2 +- crates/openshell-driver-kubernetes/src/lib.rs | 2 +- crates/openshell-driver-podman/README.md | 10 +- crates/openshell-driver-podman/src/driver.rs | 40 +- crates/openshell-driver-podman/src/watcher.rs | 11 +- crates/openshell-driver-vm/README.md | 6 +- crates/openshell-driver-vm/runtime/README.md | 2 +- crates/openshell-driver-vm/src/driver.rs | 27 +- crates/openshell-gateway/BUILD.bazel | 64 + crates/openshell-gateway/Cargo.toml | 68 + crates/openshell-gateway/src/lib.rs | 299 ++++ .../src/main.rs | 8 +- .../compute => openshell-gateway/src}/vm.rs | 70 +- crates/openshell-server/Cargo.toml | 32 +- crates/openshell-server/src/cli.rs | 227 ++- .../src/compute/driver_config.rs | 10 +- .../src/compute/driver_config/builtin.rs | 231 --- crates/openshell-server/src/compute/lease.rs | 8 +- crates/openshell-server/src/compute/mod.rs | 240 +-- crates/openshell-server/src/config_file.rs | 89 +- .../openshell-server/src/gateway_listener.rs | 96 +- crates/openshell-server/src/grpc/sandbox.rs | 42 +- crates/openshell-server/src/lib.rs | 825 ++++------- crates/openshell-server/src/otel_tracing.rs | 15 +- crates/openshell-server/src/sandbox_index.rs | 2 +- crates/openshell-server/src/tracing_setup.rs | 205 +-- deploy/docker/Dockerfile.gateway-macos | 16 +- e2e/no-compute-driver-gateway.sh | 17 +- e2e/run.sh | 4 +- e2e/rust/e2e-vm.sh | 6 +- e2e/support/gateway-common.sh | 6 +- e2e/with-kube-gateway.sh | 7 +- examples/governance-interceptor/smoke.sh | 2 +- .../smoke.sh | 2 +- tasks/ci.toml | 2 +- tasks/gateway.toml | 2 +- tasks/rust.toml | 4 +- tasks/scripts/gateway-docker.sh | 2 +- tasks/scripts/gateway-vm.sh | 2 +- tasks/scripts/package-deb-install.sh | 2 +- tasks/scripts/stage-prebuilt-binaries.sh | 2 +- .../verify-defaults-without-telemetry.sh | 2 +- tasks/scripts/vm/smoke-orphan-cleanup.sh | 2 +- 59 files changed, 1331 insertions(+), 3453 deletions(-) rename crates/openshell-core/src/{operator_namespace_allowlist.rs => dynamic_string_allowlist.rs} (84%) create mode 100644 crates/openshell-core/src/local_api_socket.rs create mode 100644 crates/openshell-gateway/BUILD.bazel create mode 100644 crates/openshell-gateway/Cargo.toml create mode 100644 crates/openshell-gateway/src/lib.rs rename crates/{openshell-server => openshell-gateway}/src/main.rs (59%) rename crates/{openshell-server/src/compute => openshell-gateway/src}/vm.rs (94%) delete mode 100644 crates/openshell-server/src/compute/driver_config/builtin.rs diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 86395af148..b73d152fc7 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -19,8 +19,9 @@ The target deployment flow is: 4. The CLI registers a reachable gateway endpoint with `openshell gateway add`. 5. The gateway creates sandboxes through the selected compute driver. -The standard gateway binary explicitly installs its compiled Docker, Podman, -Kubernetes, and VM registrations at startup. With no configured driver, the +The `openshell-gateway` composition crate explicitly installs its compiled +Docker, Podman, Kubernetes, and VM registrations at startup; `openshell-server` +does not link compute-driver crates. With no configured driver, the gateway probes only installed registrations in priority order (Kubernetes, Podman, then Docker); VM has no probe and remains opt-in. A custom gateway binary may install a different set, so confirm the binary's registered drivers diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index b779974a0a..2d22b4eafc 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -169,9 +169,9 @@ jobs: - name: Verify telemetry can be compiled out run: | - cargo build -p openshell-server --bin openshell-gateway + cargo build -p openshell-gateway --bin openshell-gateway tasks/scripts/verify-telemetry-compiled-out.sh present target/debug/openshell-gateway - cargo build -p openshell-server --bin openshell-gateway --no-default-features --features defaults-without-telemetry + cargo build -p openshell-gateway --bin openshell-gateway --no-default-features --features defaults-without-telemetry tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-gateway cargo build -p openshell-sandbox --bin openshell-sandbox --no-default-features --features defaults-without-telemetry tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-sandbox diff --git a/AGENTS.md b/AGENTS.md index 2c11b4c4ce..103e45bafc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,6 +43,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-otel-test-support/` | OpenTelemetry test support | Shared loopback OTLP collector fixture for tracing tests | | `crates/openshell-core/` | Shared core | Common types, configuration, error handling | | `crates/openshell-extension-core/` | Extension core | Shared extension identity, JWT claims, bearer-token rotation, and TLS transport primitives | +| `crates/openshell-gateway/` | Gateway binary composition | Links selected first-party compute drivers into the backend-agnostic server registry | | `crates/openshell-sdk/` | Shared client SDK | Async Rust gateway client (gRPC transport, TLS, OIDC refresh, edge tunnel); consumed by CLI, TUI, and `@openshell/sdk` | | `crates/openshell-providers/` | Provider management | Credential provider backends | | `crates/openshell-tui/` | Terminal UI | Ratatui-based dashboard for monitoring | diff --git a/README.md b/README.md index 28674cbdbf..751eba780d 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,7 @@ Disable telemetry at runtime by setting `OPENSHELL_TELEMETRY_ENABLED=false` on t You can also compile telemetry out entirely. Telemetry support is a default-on `telemetry` Cargo feature, and each crate that carries it also defines a `defaults-without-telemetry` alias covering every other default feature. Build telemetry-free artifacts with `--no-default-features --features defaults-without-telemetry`: ```shell -cargo build --release -p openshell-server --no-default-features --features defaults-without-telemetry +cargo build --release -p openshell-gateway --no-default-features --features defaults-without-telemetry cargo build --release -p openshell-sandbox --no-default-features --features defaults-without-telemetry cargo build --release -p openshell-driver-vm --no-default-features --features defaults-without-telemetry ``` diff --git a/architecture/build.md b/architecture/build.md index 7d2f9ddebd..355892abf9 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -28,7 +28,7 @@ Sandbox community images are built outside this repository. Anonymous telemetry emission is gated behind a default-on `telemetry` Cargo feature. It is defined in `openshell-core` (where the emission code, HTTP client, and endpoint live) and forwarded by the binary crates that emit or -collect telemetry: `openshell-server` (gateway), `openshell-sandbox` +collect telemetry: `openshell-gateway`, `openshell-sandbox` (supervisor), and `openshell-driver-vm`. Every crate depends on `openshell-core` with `default-features = false`, so the binary crate's feature is the single switch that enables `openshell-core/telemetry` for its build diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 200e15fe72..ff28d9899b 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -103,9 +103,8 @@ of re-querying drivers on each request. The gateway binary explicitly installs the compute drivers compiled into that binary before entering server startup. The server selects a configured driver by normalized registry name. When no driver is configured, it evaluates only -the installed drivers' probes in registered priority order, records every -available registration, and selects the first. Drivers without a probe, -including VM, remain opt-in. +the installed drivers' probes and chooses the lowest registered priority. +Drivers without a probe, including VM, remain opt-in. Startup computes this selection once after merging configuration. The same selection drives authentication defaults and runtime construction, so a probe @@ -117,17 +116,18 @@ registry. Adding or removing a compiled driver therefore changes registration rather than the server's selection flow. Alternate gateway binaries can install their own `ComputeDriverFactory` registrations and hand the completed registry to `run_cli_with_compute_drivers`; factories receive merged driver config and -finish through the same in-process runtime adapter. A configured UDS endpoint -still takes precedence over a compiled registration with the same name. - -The standard server crate groups first-party registrations behind the -`in-tree-compute-drivers` feature. Protocol-only gateway builds disable that -feature and link no compute-driver crates. E2E lanes compose that gateway with -Docker, Podman, Kubernetes, and VM driver executables over the public UDS gRPC -contract so an in-tree driver cannot silently depend on a server-only API. -External Kubernetes drivers support shared and managed workspace modes. -Operator mode requires an in-process dynamic namespace allowlist and is -rejected when Kubernetes is configured through an external endpoint. +return either an in-process driver or a gateway-managed remote endpoint. The +server constructs the common runtime adapter and snapshots `GetCapabilities` +for either result. A configured UDS endpoint still takes precedence over a +compiled registration with the same name. + +The `openshell-gateway` composition crate groups first-party registrations +behind the `in-tree-compute-drivers` feature. `openshell-server` has no compute +driver dependencies or backend-name dispatch. Protocol-only gateway builds +disable the composition feature and link no compute-driver crates. E2E lanes +compose that gateway with Docker, Podman, Kubernetes, and VM driver executables +over the public UDS gRPC contract so an in-tree driver cannot silently depend +on a server-only API. ## Stop and Start Lifecycle @@ -471,13 +471,13 @@ image-pull Secrets in every operator-managed namespace. **Operator** uses pre-provisioned namespaces discovered through two optional sources: a K8s label selector (`operator_namespace_label`) and a drop-in -allowlist file (`operator_namespace_file`). At least one must be configured. -The `OperatorNamespaceAllowlist` (`Arc>>`) is populated -at runtime by background watchers and read by the namespace resolver. Sandbox -creation fails closed if the workspace is not in the current allowlist. Platform -teams manage namespace lifecycle externally. RBAC uses the same ClusterRole as -managed mode but without namespace `create`/`delete` or ServiceAccount -permissions. +allowlist file (`operator_namespace_file`). Exactly one must be configured. +The compute driver and the gateway's ServiceAccount authenticator independently +watch that public config source; no in-process driver state crosses into the +server. Sandbox creation and token bootstrap fail closed if the workspace is +not in the current allowlist. Platform teams manage namespace lifecycle +externally. RBAC uses the same ClusterRole as managed mode but without namespace +`create`/`delete` or ServiceAccount permissions. ### Watching and Querying diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index a1f1a6c84f..3507011120 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -6,16 +6,10 @@ use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::BTreeMap; -use std::fmt; -#[cfg(unix)] -use std::io::{Read, Write}; use std::net::SocketAddr; -#[cfg(unix)] -use std::os::unix::fs::FileTypeExt; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; +use std::path::PathBuf; use std::str::FromStr; -use std::time::{Duration, Instant}; +use std::time::Duration; // ── Public default constants ──────────────────────────────────────────── // @@ -35,9 +29,6 @@ pub const DEFAULT_GATEWAY_NAME: &str = "openshell"; /// Default container stop timeout in seconds (SIGTERM → SIGKILL). pub const DEFAULT_STOP_TIMEOUT_SECS: u32 = 10; -/// Default Docker bridge network name for local sandboxes. -pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; - /// Default domain used for browser-facing sandbox service URLs. pub const DEFAULT_SERVICE_ROUTING_DOMAIN: &str = "openshell.localhost"; @@ -118,34 +109,9 @@ pub const CDI_GPU_DEVICE_ALL: &str = "nvidia.com/gpu=all"; /// Default maximum number of processes (PIDs) allowed inside a sandbox container. /// -/// Shared by the Docker and Podman drivers; override via driver config. +/// Compute drivers may override this through backend configuration. pub const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; -/// Compute backends the gateway can orchestrate sandboxes through. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ComputeDriverKind { - Kubernetes, - Vm, - Docker, - Podman, - /// Microsoft MXC isolation session (Windows only). - Mxc, -} - -impl ComputeDriverKind { - #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::Kubernetes => "kubernetes", - Self::Vm => "vm", - Self::Docker => "docker", - Self::Podman => "podman", - Self::Mxc => "mxc", - } - } -} - /// Normalize a configured compute driver name. /// /// Built-in driver names and custom remote driver names share the same @@ -167,594 +133,6 @@ pub fn normalize_compute_driver_name(value: &str) -> Result { Ok(value.to_ascii_lowercase()) } -impl fmt::Display for ComputeDriverKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -impl FromStr for ComputeDriverKind { - type Err = String; - - fn from_str(value: &str) -> Result { - match value.trim().to_ascii_lowercase().as_str() { - "kubernetes" => Ok(Self::Kubernetes), - "vm" => Ok(Self::Vm), - "docker" => Ok(Self::Docker), - "podman" => Ok(Self::Podman), - "mxc" => Ok(Self::Mxc), - other => Err(format!( - "unsupported compute driver '{other}'. expected one of: kubernetes, vm, docker, podman, mxc" - )), - } - } -} - -/// Auto-detect the appropriate compute driver based on the runtime environment. -/// -/// Priority order: Kubernetes → Podman → Docker. -/// VM is never auto-detected (requires explicit `--drivers vm`). -/// -/// Returns the first driver where the environment check passes. -/// Returns `None` if no compatible driver is found. -pub fn detect_driver() -> Option { - // Kubernetes: check for KUBERNETES_SERVICE_HOST env var (set inside pods) - if std::env::var_os("KUBERNETES_SERVICE_HOST").is_some() { - return Some(ComputeDriverKind::Kubernetes); - } - - // Podman: check for a reachable local API socket. - if is_podman_available() { - return Some(ComputeDriverKind::Podman); - } - - // Docker: check for a reachable local API socket. - if is_docker_available() { - return Some(ComputeDriverKind::Docker); - } - - None -} - -/// Return whether a responsive local Podman API socket is available. -#[must_use] -pub fn is_podman_available() -> bool { - detect_podman_socket().is_some() -} - -/// Return the Podman API socket, or `None` if Podman is not available. -/// -/// Probes the well-known socket candidates first, then falls back to asking -/// the Podman CLI where its socket lives. The symlink at a well-known path is -/// not always present — it varies by Podman version, machine provider, and -/// platform — so the CLI fallback is what makes detection work on hosts where -/// Podman is functional but the socket is somewhere else. -pub fn detect_podman_socket() -> Option { - detect_podman_socket_from_candidates(&podman_socket_candidates()) - .or_else(discover_podman_socket) -} - -fn detect_podman_socket_from_candidates(candidates: &[PathBuf]) -> Option { - candidates - .iter() - .find(|path| podman_socket_responds(path)) - .cloned() -} - -/// Maximum time to wait for a Podman discovery subprocess. -/// -/// Driver auto-detection runs `podman info`, `podman machine inspect`, and -/// `podman system connection list` during gateway startup. A stalled Podman -/// machine, SSH connection, provider, or helper must not block startup forever, -/// so each probe is bounded and treated as "not found" on expiry — detection -/// then continues to the next candidate driver or fails with an actionable -/// error. -const PODMAN_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(5); - -/// How often the bounded runner polls the child for completion. -const PODMAN_DISCOVERY_POLL_INTERVAL: Duration = Duration::from_millis(25); - -/// Run `podman ` with a bounded deadline, returning captured stdout on a -/// successful exit. Returns `None` on spawn failure, non-zero exit, or timeout. -fn run_podman_capture(args: &[&str]) -> Option> { - run_bounded_command("podman", args, PODMAN_DISCOVERY_TIMEOUT) -} - -/// Run `program ` with a deadline, capturing stdout. -/// -/// Unlike `Command::output()`, which blocks until the child exits, the deadline -/// is absolute: the call returns within `timeout` no matter what the probe or -/// its descendants do. A Podman probe can leave a daemonized descendant (an SSH -/// multiplexer, `gvproxy`, etc.) that inherited the stdout pipe, so -/// `read_to_end` would otherwise wait for EOF forever even after the direct -/// child exits — and such a descendant may even have escaped the probe's -/// process group. To stay bounded, the deadline covers both waiting for the -/// child and draining its stdout; on expiry the call best-effort kills the -/// process group (cleaning up in-group descendants) and gives up immediately, -/// abandoning the reader thread rather than waiting on it again. The abandoned -/// reader exits on its own once the pipe finally closes. Returns the captured -/// stdout only on a successful exit whose output was fully drained within the -/// deadline; otherwise `None`. -fn run_bounded_command(program: &str, args: &[&str], timeout: Duration) -> Option> { - use std::io::Read as _; - use std::sync::mpsc; - - let mut command = Command::new(program); - command - .args(args) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()); - // Run the probe as its own process-group leader so in-group descendants that - // inherited the stdout pipe can be terminated as a group on timeout. - set_new_process_group(&mut command); - let mut child = command.spawn().ok()?; - - // Drain stdout on a separate thread so a child that fills the pipe buffer - // cannot deadlock against the polling loop below. The reader reports through - // a channel so the drain can be bounded by the deadline and abandoned if it - // outlives it. - let mut stdout = child.stdout.take()?; - let (tx, rx) = mpsc::channel(); - std::thread::spawn(move || { - let mut buf = Vec::new(); - let _ = stdout.read_to_end(&mut buf); - let _ = tx.send(buf); - }); - - let deadline = Instant::now() + timeout; - - // Wait for the direct child to exit, bounded by the deadline. - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break Some(status), - Ok(None) => { - if Instant::now() >= deadline { - break None; - } - std::thread::sleep(PODMAN_DISCOVERY_POLL_INTERVAL); - } - Err(_) => break None, - } - }; - - // The child never exited within the deadline: kill the group, reap the - // (now-killed) direct child, and give up. `wait()` is bounded because the - // direct child is dead. - let Some(status) = status else { - terminate_process_group(&mut child); - let _ = child.wait(); - return None; - }; - - // The direct child exited (already reaped by `try_wait`). Collect its stdout - // without ever blocking past the deadline. A surviving descendant — possibly - // one that escaped the process group — can hold the pipe open indefinitely, - // so on expiry best-effort kill the group for cleanup and return None rather - // than waiting on the reader again. - let remaining = deadline.saturating_duration_since(Instant::now()); - match rx.recv_timeout(remaining) { - Ok(stdout) if status.success() => Some(stdout), - Ok(_) => None, - Err(_) => { - terminate_process_group(&mut child); - None - } - } -} - -/// Configure `command` to start its child as a new process-group leader so the -/// group can be signaled as a unit. No-op on non-Unix platforms. -#[cfg(unix)] -fn set_new_process_group(command: &mut Command) { - use std::os::unix::process::CommandExt as _; - command.process_group(0); -} - -#[cfg(not(unix))] -fn set_new_process_group(_command: &mut Command) {} - -/// Kill the child's whole process group so daemonized descendants that inherited -/// the stdout pipe are terminated too. Falls back to killing just the child. -#[cfg(unix)] -fn terminate_process_group(child: &mut std::process::Child) { - // `set_new_process_group` made the child a group leader, so its PID doubles - // as the group ID; signaling the negated PID targets the entire group. The - // group stays valid while a descendant is alive, so this reaches survivors - // even after the leader has been reaped. - let raw_pid = i32::try_from(child.id()).unwrap_or(i32::MAX); - let pgid = nix::unistd::Pid::from_raw(-raw_pid); - let _ = nix::sys::signal::kill(pgid, nix::sys::signal::Signal::SIGKILL); - let _ = child.kill(); -} - -#[cfg(not(unix))] -fn terminate_process_group(child: &mut std::process::Child) { - let _ = child.kill(); -} - -/// Query the Podman CLI to discover the host-side API socket path. -/// -/// Strategy: -/// 1. Run `podman info --format json` to check connectivity and whether -/// the service is remote (macOS/Windows VM) or local (native Linux). -/// 2. If `CONTAINER_HOST` explicitly points at a Unix socket, `podman info` -/// just connected through it — use that path directly (a raw unix:// URL -/// has no machine to inspect and reports the VM-internal socket). -/// 3. If `serviceIsRemote` is true, run `podman machine inspect` to get -/// the host-side forwarded socket (the `remoteSocket` from `podman info` -/// is the VM-internal path, which is not reachable from the host). -/// 4. If `serviceIsRemote` is false, use `remoteSocket.path` directly -/// (on native Linux this IS the real local socket). -fn discover_podman_socket() -> Option { - let stdout = run_podman_capture(&["info", "--format", "json"])?; - - // podman info succeeded, so an explicit unix:// CONTAINER_HOST is the exact - // working host-side socket. This must be checked before the machine path, - // which cannot map a raw unix:// endpoint to a machine. - if let Some(path) = explicit_unix_container_host() { - return Some(path); - } - - let info: serde_json::Value = serde_json::from_slice(&stdout).ok()?; - let is_remote = info["host"]["serviceIsRemote"].as_bool().unwrap_or(false); - - if is_remote { - discover_podman_machine_socket() - } else { - parse_podman_info_socket(&info) - } -} - -/// Return the socket path when `CONTAINER_HOST` is an explicit `unix://` URL. -/// -/// Honors Podman's precedence: `CONTAINER_CONNECTION` outranks `CONTAINER_HOST`, -/// so a set `CONTAINER_CONNECTION` means `podman info` did not use -/// `CONTAINER_HOST` and this returns `None`. -fn explicit_unix_container_host() -> Option { - if env_var_nonempty("CONTAINER_CONNECTION").is_some() { - return None; - } - let host = env_var_nonempty("CONTAINER_HOST")?; - unix_url_socket_path(&host) -} - -/// Parse the socket path from a `unix://` URL, or `None` for other schemes. -fn unix_url_socket_path(url: &str) -> Option { - let path = url.trim().strip_prefix("unix://")?; - (!path.is_empty()).then(|| PathBuf::from(path)) -} - -/// Extract the socket path from `podman info` JSON output. -/// Used on native Linux where `remoteSocket.path` is the real local socket. -fn parse_podman_info_socket(info: &serde_json::Value) -> Option { - let path_str = info["host"]["remoteSocket"]["path"].as_str()?; - let path = path_str.strip_prefix("unix://").unwrap_or(path_str); - if path.is_empty() { - return None; - } - Some(PathBuf::from(path)) -} - -/// Which Podman machine `podman info` connected through. -/// -/// Podman resolves its endpoint (highest precedence first) from -/// `CONTAINER_CONNECTION` (a named connection), then `CONTAINER_HOST` (a URL), -/// then the default connection in `containers.conf`. -#[derive(Debug, PartialEq, Eq)] -enum ActiveMachine { - /// An explicit selector (`CONTAINER_CONNECTION`, or `CONTAINER_HOST` mapped - /// to a connection by URL) named this connection. It must match a machine - /// exactly; guessing another machine would connect to the wrong socket. - Explicit(String), - /// An explicit `CONTAINER_HOST` is set but maps to no known connection - /// (e.g. a plain remote server, not a local machine). The active machine - /// cannot be determined and must not be guessed. - UnresolvedExplicit, - /// No explicit selector; the `containers.conf` default connection name, if - /// any. When absent, Podman's built-in default machine is inspected by name. - Default(Option), -} - -/// Run `podman machine inspect ` to discover the host-side forwarded -/// socket. Used on macOS/Windows where the Podman service runs inside a VM. -/// -/// The active machine is resolved first and inspected *by name*: a no-argument -/// `podman machine inspect` inspects only `podman-machine-default`, so a host -/// whose default connection is a different machine would otherwise be pointed -/// at the wrong machine's socket. When the active machine cannot be mapped to a -/// name, this returns `None` rather than substituting an unrelated machine. -fn discover_podman_machine_socket() -> Option { - let targets = podman_machine_inspect_targets(&active_podman_machine())?; - targets.iter().find_map(|name| { - let stdout = run_podman_capture(&["machine", "inspect", name])?; - let machines: serde_json::Value = serde_json::from_slice(&stdout).ok()?; - parse_podman_machine_inspect_socket(&machines) - }) -} - -/// Machine names to try with `podman machine inspect`, most specific first. -/// -/// `None` means the active machine cannot be determined; inspection must not -/// guess, since picking an unrelated machine would return the wrong socket. A -/// rootful connection is named `-root` while the machine itself is -/// ``, so the `-root`-stripped name is offered as a fallback. -fn podman_machine_inspect_targets(active: &ActiveMachine) -> Option> { - fn names_for(connection: &str) -> Vec { - let mut names = vec![connection.to_string()]; - if let Some(stripped) = connection.strip_suffix("-root") - && !stripped.is_empty() - { - names.push(stripped.to_string()); - } - names - } - - match active { - ActiveMachine::Explicit(name) | ActiveMachine::Default(Some(name)) => Some(names_for(name)), - ActiveMachine::UnresolvedExplicit => None, - // No explicit selector and no default connection: `podman info` used - // Podman's built-in default machine, so inspect it by name rather than - // guessing an arbitrary entry. - ActiveMachine::Default(None) => Some(vec!["podman-machine-default".to_string()]), - } -} - -/// Determine which machine `podman info` connected through. -fn active_podman_machine() -> ActiveMachine { - if let Some(name) = env_var_nonempty("CONTAINER_CONNECTION") { - return ActiveMachine::Explicit(name); - } - let container_host = env_var_nonempty("CONTAINER_HOST"); - resolve_active_podman_machine(container_host.as_deref(), podman_connection_list().as_ref()) -} - -/// Resolve the active machine from `CONTAINER_HOST` and the connection list. -/// -/// `CONTAINER_CONNECTION` is handled by the caller (it needs no connection -/// list). This is split out as a pure function for testing. -fn resolve_active_podman_machine( - container_host: Option<&str>, - connections: Option<&serde_json::Value>, -) -> ActiveMachine { - if let Some(host) = container_host { - return connections - .and_then(|c| podman_connection_name_for_uri(c, host)) - .map_or(ActiveMachine::UnresolvedExplicit, ActiveMachine::Explicit); - } - ActiveMachine::Default(connections.and_then(parse_default_podman_connection)) -} - -fn env_var_nonempty(key: &str) -> Option { - std::env::var(key) - .ok() - .filter(|value| !value.trim().is_empty()) -} - -/// Run `podman system connection list --format json`. -fn podman_connection_list() -> Option { - let stdout = run_podman_capture(&["system", "connection", "list", "--format", "json"])?; - serde_json::from_slice(&stdout).ok() -} - -/// Extract the default machine connection name from -/// `podman system connection list --format json`. -fn parse_default_podman_connection(connections: &serde_json::Value) -> Option { - connections - .as_array()? - .iter() - .find(|c| { - c["Default"].as_bool().unwrap_or(false) && c["IsMachine"].as_bool().unwrap_or(false) - }) - .and_then(|c| c["Name"].as_str()) - .map(str::to_string) -} - -/// Find the machine connection whose URI matches `CONTAINER_HOST`. -/// -/// Only machine connections (`IsMachine: true`) map to a local socket, so a -/// `CONTAINER_HOST` pointing at a plain remote server yields `None`. -fn podman_connection_name_for_uri(connections: &serde_json::Value, uri: &str) -> Option { - connections - .as_array()? - .iter() - .find(|c| c["IsMachine"].as_bool().unwrap_or(false) && c["URI"].as_str() == Some(uri)) - .and_then(|c| c["Name"].as_str()) - .map(str::to_string) -} - -/// Extract the host-side socket path from a `podman machine inspect ` -/// JSON array (which contains only the inspected machine). -fn parse_podman_machine_inspect_socket(machines: &serde_json::Value) -> Option { - let machine = machines.as_array()?.first()?; - let path_str = machine["ConnectionInfo"]["PodmanSocket"]["Path"].as_str()?; - if path_str.is_empty() { - return None; - } - Some(PathBuf::from(path_str)) -} - -fn podman_socket_candidates() -> Vec { - let socket = std::env::var("OPENSHELL_PODMAN_SOCKET") - .ok() - .filter(|path| !path.trim().is_empty()) - .map(PathBuf::from); - podman_socket_candidates_from_env( - socket, - std::env::var_os("XDG_RUNTIME_DIR").map(PathBuf::from), - std::env::var_os("HOME").map(PathBuf::from), - ) -} - -fn podman_socket_candidates_from_env( - socket: Option, - runtime_dir: Option, - home: Option, -) -> Vec { - let mut candidates = Vec::new(); - - if let Some(path) = socket { - candidates.push(path); - } - - if let Some(runtime_dir) = runtime_dir { - candidates.push(runtime_dir.join("podman/podman.sock")); - } - - #[cfg(target_os = "linux")] - { - candidates.push(PathBuf::from(format!( - "/run/user/{}/podman/podman.sock", - current_uid() - ))); - } - - if let Some(home) = home { - candidates.push(home.join(".local/share/containers/podman/machine/podman.sock")); - } - - candidates -} - -/// Return whether a responsive local Docker API socket is available. -#[must_use] -pub fn is_docker_available() -> bool { - detect_docker_socket().is_some() -} - -pub fn detect_docker_socket() -> Option { - detect_docker_socket_from_candidates(&docker_socket_candidates()) -} - -fn detect_docker_socket_from_candidates(candidates: &[PathBuf]) -> Option { - candidates - .iter() - .find(|path| docker_socket_responds(path)) - .cloned() -} - -fn docker_socket_candidates() -> Vec { - let mut candidates = Vec::new(); - - if let Ok(host) = std::env::var("DOCKER_HOST") - && let Some(path) = docker_host_unix_socket_path(&host) - { - candidates.push(path); - } - - candidates.push(PathBuf::from("/var/run/docker.sock")); - - if let Some(home) = std::env::var_os("HOME") { - candidates.push(PathBuf::from(home).join(".docker/run/docker.sock")); - } - - if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { - candidates.push(PathBuf::from(runtime_dir).join("docker.sock")); - } - - candidates -} - -fn docker_host_unix_socket_path(host: &str) -> Option { - let path = host.trim().strip_prefix("unix://")?; - (!path.is_empty()).then(|| PathBuf::from(path)) -} - -#[cfg(unix)] -fn is_unix_socket(path: &Path) -> bool { - path.metadata() - .is_ok_and(|metadata| metadata.file_type().is_socket()) -} - -#[cfg(unix)] -fn podman_socket_responds(path: &Path) -> bool { - unix_socket_http_ping(path, |response| { - http_response_is_success(response) && contains_ascii(response, b"Libpod-Api-Version:") - }) -} - -#[cfg(unix)] -fn docker_socket_responds(path: &Path) -> bool { - unix_socket_http_ping(path, |response| { - http_response_is_success(response) - && contains_ascii(response, b"Api-Version:") - && !contains_ascii(response, b"Libpod-Api-Version:") - }) -} - -#[cfg(unix)] -fn unix_socket_http_ping(path: &Path, accepts_response: impl FnOnce(&[u8]) -> bool) -> bool { - const PROBE_TIMEOUT: Duration = Duration::from_secs(1); - const PING_REQUEST: &[u8] = - b"GET /_ping HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; - - if !is_unix_socket(path) { - return false; - } - - let Ok(mut stream) = std::os::unix::net::UnixStream::connect(path) else { - return false; - }; - if stream.set_read_timeout(Some(PROBE_TIMEOUT)).is_err() - || stream.set_write_timeout(Some(PROBE_TIMEOUT)).is_err() - || stream.write_all(PING_REQUEST).is_err() - { - return false; - } - - let mut response = [0_u8; 512]; - let mut total = 0; - while total < response.len() { - let Ok(n) = stream.read(&mut response[total..]) else { - return false; - }; - if n == 0 { - break; - } - total += n; - if contains_ascii(&response[..total], b"\r\n\r\n") { - break; - } - } - total > 0 && accepts_response(&response[..total]) -} - -#[cfg(unix)] -fn http_response_is_success(response: &[u8]) -> bool { - response.starts_with(b"HTTP/1.1 200") || response.starts_with(b"HTTP/1.0 200") -} - -#[cfg(unix)] -fn contains_ascii(haystack: &[u8], needle: &[u8]) -> bool { - haystack - .windows(needle.len()) - .any(|window| window.eq_ignore_ascii_case(needle)) -} - -#[cfg(all(unix, test))] -fn is_reachable_unix_socket(path: &Path) -> bool { - is_unix_socket(path) && std::os::unix::net::UnixStream::connect(path).is_ok() -} - -#[cfg(all(unix, target_os = "linux"))] -fn current_uid() -> u32 { - use std::os::unix::fs::MetadataExt; - - std::fs::metadata("/proc/self").map_or(0, |metadata| metadata.uid()) -} - -#[cfg(not(unix))] -fn podman_socket_responds(path: &Path) -> bool { - let _ = path; - false -} - -#[cfg(not(unix))] -fn docker_socket_responds(path: &Path) -> bool { - let _ = path; - false -} - /// Server configuration. /// /// Built programmatically in [`crate::Config::new`] and the gateway CLI from @@ -1447,53 +825,13 @@ const fn default_ssh_session_ttl_secs() -> u64 { #[cfg(test)] mod tests { use super::{ - ActiveMachine, ComputeDriverKind, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, - GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, - GatewayJwtConfig, GatewayProviderProfileSourceConfig, PolicyValidationFailureMode, - detect_docker_socket_from_candidates, detect_driver, detect_podman_socket_from_candidates, - docker_host_unix_socket_path, docker_socket_responds, explicit_unix_container_host, - normalize_compute_driver_name, parse_default_podman_connection, parse_podman_info_socket, - parse_podman_machine_inspect_socket, podman_connection_name_for_uri, - podman_machine_inspect_targets, podman_socket_candidates_from_env, podman_socket_responds, - resolve_active_podman_machine, run_bounded_command, unix_url_socket_path, + Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, + GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayJwtConfig, + GatewayProviderProfileSourceConfig, PolicyValidationFailureMode, + normalize_compute_driver_name, }; - #[cfg(unix)] - use super::{is_reachable_unix_socket, is_unix_socket}; - #[cfg(unix)] - use std::io::{Read as _, Write as _}; use std::net::SocketAddr; - #[cfg(unix)] - use std::os::unix::net::UnixListener; - use std::path::PathBuf; use std::time::Duration; - #[cfg(unix)] - use std::time::Instant; - - #[test] - fn compute_driver_kind_parses_supported_values() { - assert_eq!( - "kubernetes".parse::().unwrap(), - ComputeDriverKind::Kubernetes - ); - assert_eq!( - "vm".parse::().unwrap(), - ComputeDriverKind::Vm - ); - assert_eq!( - "podman".parse::().unwrap(), - ComputeDriverKind::Podman - ); - assert_eq!( - "docker".parse::().unwrap(), - ComputeDriverKind::Docker - ); - } - - #[test] - fn compute_driver_kind_rejects_unknown_values() { - let err = "firecracker".parse::().unwrap_err(); - assert!(err.contains("unsupported compute driver 'firecracker'")); - } #[test] fn policy_validation_failure_mode_is_secure_by_default() { @@ -1714,244 +1052,6 @@ mod tests { assert_eq!(cfg.health_bind_address, Some(addr)); } - #[test] - fn detect_driver_returns_none_without_k8s_env_or_local_runtime() { - // When KUBERNETES_SERVICE_HOST is not set, no Docker binary/socket is - // available, and no Podman API socket is available, detect_driver - // should return None. - // This test may pass or fail depending on the test environment, - // but it documents the expected behavior. - let _ = detect_driver(); // Returns Some or None based on environment - } - - #[test] - fn docker_host_unix_socket_path_parses_unix_hosts() { - assert_eq!( - docker_host_unix_socket_path("unix:///var/run/docker.sock"), - Some(PathBuf::from("/var/run/docker.sock")) - ); - assert_eq!(docker_host_unix_socket_path("tcp://127.0.0.1:2375"), None); - assert_eq!(docker_host_unix_socket_path("unix://"), None); - } - - #[cfg(unix)] - #[test] - fn is_unix_socket_detects_socket_files() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("docker.sock"); - let _listener = UnixListener::bind(&socket_path).expect("bind unix socket"); - - assert!(is_unix_socket(&socket_path)); - assert!(is_reachable_unix_socket(&socket_path)); - - let regular_file = temp_dir.path().join("not-a-socket"); - std::fs::write(®ular_file, b"not a socket").expect("write regular file"); - assert!(!is_unix_socket(®ular_file)); - assert!(!is_reachable_unix_socket(®ular_file)); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn podman_socket_probe_accepts_successful_ping_response() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("podman.sock"); - let listener = UnixListener::bind(&socket_path).expect("bind podman socket"); - - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept podman probe"); - let mut request = [0_u8; 128]; - let n = stream.read(&mut request).expect("read podman probe"); - assert!(request[..n].starts_with(b"GET /_ping HTTP/1.1\r\n")); - stream - .write_all( - b"HTTP/1.1 200 OK\r\nLibpod-Api-Version: 5.8.2\r\nContent-Length: 2\r\n\r\nOK", - ) - .expect("write podman ping response"); - }); - - assert!(podman_socket_responds(&socket_path)); - handle.join().expect("probe server exits"); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn podman_socket_probe_rejects_docker_ping_response() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("podman.sock"); - let listener = UnixListener::bind(&socket_path).expect("bind podman socket"); - - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept podman probe"); - let mut request = [0_u8; 128]; - let n = stream.read(&mut request).expect("read podman probe"); - assert!(request[..n].starts_with(b"GET /_ping HTTP/1.1\r\n")); - stream - .write_all( - b"HTTP/1.1 200 OK\r\nServer: Docker/29.2.1\r\nContent-Length: 2\r\n\r\nOK", - ) - .expect("write docker ping response"); - }); - - assert!(!podman_socket_responds(&socket_path)); - handle.join().expect("probe server exits"); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn docker_socket_probe_accepts_successful_ping_response() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("docker.sock"); - let listener = UnixListener::bind(&socket_path).expect("bind docker socket"); - - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept docker probe"); - let mut request = [0_u8; 128]; - let n = stream.read(&mut request).expect("read docker probe"); - assert!(request[..n].starts_with(b"GET /_ping HTTP/1.1\r\n")); - stream - .write_all( - b"HTTP/1.1 200 OK\r\nApi-Version: 1.51\r\nDocker-Experimental: false\r\nContent-Length: 2\r\n\r\nOK", - ) - .expect("write docker ping response"); - }); - - assert!(docker_socket_responds(&socket_path)); - handle.join().expect("probe server exits"); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn docker_socket_probe_rejects_podman_ping_response() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("podman.sock"); - let listener = UnixListener::bind(&socket_path).expect("bind podman socket"); - - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept docker probe"); - let mut request = [0_u8; 128]; - let n = stream.read(&mut request).expect("read docker probe"); - assert!(request[..n].starts_with(b"GET /_ping HTTP/1.1\r\n")); - stream - .write_all( - b"HTTP/1.1 200 OK\r\nLibpod-Api-Version: 5.8.2\r\nContent-Length: 2\r\n\r\nOK", - ) - .expect("write podman ping response"); - }); - - assert!(!docker_socket_responds(&socket_path)); - handle.join().expect("probe server exits"); - } - - #[cfg(unix)] - #[test] - fn docker_socket_probe_rejects_inactive_socket() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("docker.sock"); - let listener = UnixListener::bind(&socket_path).expect("bind docker socket"); - drop(listener); - - assert!(is_unix_socket(&socket_path)); - assert!(!docker_socket_responds(&socket_path)); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn docker_socket_detection_returns_the_responsive_candidate() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let inactive_path = temp_dir.path().join("inactive.sock"); - let inactive_listener = UnixListener::bind(&inactive_path).expect("bind inactive socket"); - drop(inactive_listener); - - let responsive_path = temp_dir.path().join("responsive.sock"); - let listener = UnixListener::bind(&responsive_path).expect("bind responsive socket"); - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept docker probe"); - let mut request = [0_u8; 128]; - let _ = stream.read(&mut request).expect("read docker probe"); - stream - .write_all(b"HTTP/1.1 200 OK\r\nApi-Version: 1.51\r\nContent-Length: 2\r\n\r\nOK") - .expect("write docker ping response"); - }); - - assert_eq!( - detect_docker_socket_from_candidates(&[inactive_path, responsive_path.clone(),]), - Some(responsive_path) - ); - handle.join().expect("probe server exits"); - } - - #[test] - fn podman_socket_candidates_include_env_runtime_and_home_paths() { - let candidates = podman_socket_candidates_from_env( - Some(PathBuf::from("/tmp/custom-podman.sock")), - Some(PathBuf::from("/tmp/runtime")), - Some(PathBuf::from("/tmp/home")), - ); - - assert!(candidates.contains(&PathBuf::from("/tmp/custom-podman.sock"))); - assert!(candidates.contains(&PathBuf::from("/tmp/runtime/podman/podman.sock"))); - assert!(candidates.contains(&PathBuf::from( - "/tmp/home/.local/share/containers/podman/machine/podman.sock" - ))); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn podman_socket_detection_returns_the_responsive_candidate() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let inactive_path = temp_dir.path().join("inactive.sock"); - let inactive_listener = UnixListener::bind(&inactive_path).expect("bind inactive socket"); - drop(inactive_listener); - - let responsive_path = temp_dir.path().join("responsive.sock"); - let listener = UnixListener::bind(&responsive_path).expect("bind responsive socket"); - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept podman probe"); - let mut request = [0_u8; 128]; - let _ = stream.read(&mut request).expect("read podman probe"); - stream - .write_all( - b"HTTP/1.1 200 OK\r\nLibpod-Api-Version: 5.8.2\r\nContent-Length: 2\r\n\r\nOK", - ) - .expect("write podman ping response"); - }); - - assert_eq!( - detect_podman_socket_from_candidates(&[inactive_path, responsive_path.clone(),]), - Some(responsive_path) - ); - handle.join().expect("probe server exits"); - } - - #[test] - #[allow(unsafe_code)] // std::env::set_var/remove_var require unsafe in Rust 2024 - fn detect_driver_prefers_kubernetes_when_k8s_env_is_set() { - // Save the original env var - let original = std::env::var("KUBERNETES_SERVICE_HOST").ok(); - - // Set the env var - unsafe { - std::env::set_var("KUBERNETES_SERVICE_HOST", "127.0.0.1"); - } - - let result = detect_driver(); - assert_eq!(result, Some(ComputeDriverKind::Kubernetes)); - - // Restore the original env var - unsafe { - match original { - Some(val) => std::env::set_var("KUBERNETES_SERVICE_HOST", val), - None => std::env::remove_var("KUBERNETES_SERVICE_HOST"), - } - } - } - #[test] fn supervisor_image_tag_prefers_explicit_build_tags() { use super::resolve_supervisor_image_tag; @@ -1968,23 +1068,6 @@ mod tests { ); } - #[test] - fn parse_podman_info_socket_extracts_linux_local_socket() { - let info: serde_json::Value = serde_json::json!({ - "host": { - "serviceIsRemote": false, - "remoteSocket": { - "path": "unix:///run/user/1000/podman/podman.sock", - "exists": true - } - } - }); - assert_eq!( - parse_podman_info_socket(&info), - Some(PathBuf::from("/run/user/1000/podman/podman.sock")) - ); - } - #[test] fn supervisor_image_tag_sanitizes_build_metadata_for_oci() { use super::resolve_supervisor_image_tag; @@ -1998,22 +1081,6 @@ mod tests { ); } - #[test] - fn parse_podman_info_socket_handles_path_without_unix_prefix() { - let info: serde_json::Value = serde_json::json!({ - "host": { - "remoteSocket": { - "path": "/run/user/1000/podman/podman.sock", - "exists": true - } - } - }); - assert_eq!( - parse_podman_info_socket(&info), - Some(PathBuf::from("/run/user/1000/podman/podman.sock")) - ); - } - #[test] fn default_supervisor_image_is_version_pinned() { use super::default_supervisor_image; @@ -2022,348 +1089,4 @@ mod tests { let tag = image.rsplit_once(':').unwrap().1; assert!(!tag.is_empty()); } - - #[test] - fn parse_podman_info_socket_returns_none_for_missing_path() { - let info: serde_json::Value = serde_json::json!({ - "host": { - "remoteSocket": {} - } - }); - assert_eq!(parse_podman_info_socket(&info), None); - } - - #[test] - fn parse_podman_info_socket_returns_none_for_empty_path() { - let info: serde_json::Value = serde_json::json!({ - "host": { - "remoteSocket": { - "path": "", - "exists": false - } - } - }); - assert_eq!(parse_podman_info_socket(&info), None); - } - - #[test] - fn parse_podman_machine_inspect_socket_extracts_macos_socket() { - // `podman machine inspect ` returns only the inspected machine. - let machines: serde_json::Value = serde_json::json!([ - { - "ConnectionInfo": { - "PodmanSocket": { - "Path": "/var/folders/1q/jx7s14b928n8zvstgfk98lj00000gn/T/podman/podman-machine-default-api.sock" - }, - "PodmanPipe": null - }, - "Name": "podman-machine-default" - } - ]); - assert_eq!( - parse_podman_machine_inspect_socket(&machines), - Some(PathBuf::from( - "/var/folders/1q/jx7s14b928n8zvstgfk98lj00000gn/T/podman/podman-machine-default-api.sock" - )) - ); - } - - #[test] - fn parse_podman_machine_inspect_socket_returns_none_for_empty_array() { - let machines: serde_json::Value = serde_json::json!([]); - assert_eq!(parse_podman_machine_inspect_socket(&machines), None); - } - - #[test] - fn parse_podman_machine_inspect_socket_returns_none_for_missing_socket() { - let machines: serde_json::Value = serde_json::json!([ - { - "ConnectionInfo": {}, - "Name": "podman-machine-default" - } - ]); - assert_eq!(parse_podman_machine_inspect_socket(&machines), None); - } - - #[test] - fn podman_machine_inspect_targets_uses_explicit_machine_by_name() { - // The active connection points at `work`; discovery must inspect `work` - // explicitly rather than the no-argument default machine, which would - // return a different machine's socket. - assert_eq!( - podman_machine_inspect_targets(&ActiveMachine::Explicit("work".to_string())), - Some(vec!["work".to_string()]) - ); - } - - #[test] - fn podman_machine_inspect_targets_strips_rootful_suffix() { - // Rootful connections are named `-root`; the machine itself is - // ``, offered as a fallback after the connection name. - assert_eq!( - podman_machine_inspect_targets(&ActiveMachine::Explicit("work-root".to_string())), - Some(vec!["work-root".to_string(), "work".to_string()]) - ); - } - - #[test] - fn podman_machine_inspect_targets_uses_default_connection_name() { - assert_eq!( - podman_machine_inspect_targets(&ActiveMachine::Default(Some("work".to_string()))), - Some(vec!["work".to_string()]) - ); - } - - #[test] - fn podman_machine_inspect_targets_falls_back_to_builtin_default() { - // No explicit selector and no default connection: inspect Podman's own - // built-in default machine by name. - assert_eq!( - podman_machine_inspect_targets(&ActiveMachine::Default(None)), - Some(vec!["podman-machine-default".to_string()]) - ); - } - - #[test] - fn podman_machine_inspect_targets_does_not_guess_for_unresolved_explicit() { - // CONTAINER_HOST pointing at a non-machine endpoint cannot be mapped to - // a machine; discovery must not guess an unrelated one. - assert_eq!( - podman_machine_inspect_targets(&ActiveMachine::UnresolvedExplicit), - None - ); - } - - #[cfg(unix)] - #[test] - fn run_bounded_command_captures_stdout_on_success() { - assert_eq!( - run_bounded_command("printf", &["hello"], Duration::from_secs(5)), - Some(b"hello".to_vec()) - ); - } - - #[cfg(unix)] - #[test] - fn run_bounded_command_returns_none_on_nonzero_exit() { - assert_eq!( - run_bounded_command("false", &[], Duration::from_secs(5)), - None - ); - } - - #[cfg(unix)] - #[test] - fn run_bounded_command_kills_child_that_exceeds_deadline() { - // A process that would otherwise block startup indefinitely must be - // bounded: `run_bounded_command` returns within the deadline instead of - // hanging until the child exits. - let start = Instant::now(); - let result = run_bounded_command("sleep", &["30"], Duration::from_millis(200)); - let elapsed = start.elapsed(); - assert_eq!(result, None); - assert!( - elapsed < Duration::from_secs(5), - "bounded command did not return promptly: {elapsed:?}" - ); - } - - #[cfg(unix)] - #[test] - fn run_bounded_command_bounds_drain_when_in_group_descendant_holds_stdout() { - // The shell exits immediately after `echo`, but the backgrounded child - // (in the same process group) inherits and holds the stdout pipe open. - // Without a bounded drain, `read_to_end` would wait ~30s for EOF even - // though the direct child already exited. - let start = Instant::now(); - let result = run_bounded_command( - "sh", - &["-c", "sleep 30 & echo done"], - Duration::from_millis(300), - ); - let elapsed = start.elapsed(); - assert!( - elapsed < Duration::from_secs(2), - "drain blocked on a descendant holding stdout: {elapsed:?}" - ); - // Draining hit the deadline, so the probe is treated as "not found". - assert_eq!(result, None); - } - - #[cfg(unix)] - #[test] - fn run_bounded_command_bounds_drain_when_descendant_escapes_process_group() { - // `set -m` runs the background job in its OWN process group, so it - // survives the group kill while still holding the stdout pipe. The - // deadline must remain absolute: the call must not fall back to an - // untimed receive that waits for the escaped descendant's EOF. - let start = Instant::now(); - let result = run_bounded_command( - "bash", - &["-c", "set -m; sleep 5 & echo done"], - Duration::from_millis(300), - ); - let elapsed = start.elapsed(); - // A regression (blocking receive after the timeout) would wait ~5s for - // the escaped `sleep`; the bounded implementation returns promptly. - assert!( - elapsed < Duration::from_secs(2), - "drain blocked on a descendant that escaped the process group: {elapsed:?}" - ); - assert_eq!(result, None); - } - - #[cfg(unix)] - #[test] - fn run_bounded_command_returns_none_for_missing_program() { - assert_eq!( - run_bounded_command( - "openshell-nonexistent-binary-xyz", - &[], - Duration::from_secs(5) - ), - None - ); - } - - #[test] - fn resolve_active_podman_machine_maps_container_host_to_connection() { - let connections: serde_json::Value = serde_json::json!([ - { "Name": "work", "IsMachine": true, "Default": false, - "URI": "ssh://core@127.0.0.1:5555/run/user/1000/podman/podman.sock" }, - { "Name": "podman-machine-default", "IsMachine": true, "Default": true, - "URI": "ssh://core@127.0.0.1:4444/run/user/1000/podman/podman.sock" } - ]); - // CONTAINER_HOST pointing at the non-default machine's URI resolves to - // that machine, not the default. - assert_eq!( - resolve_active_podman_machine( - Some("ssh://core@127.0.0.1:5555/run/user/1000/podman/podman.sock"), - Some(&connections) - ), - ActiveMachine::Explicit("work".to_string()) - ); - } - - #[test] - fn resolve_active_podman_machine_unmatched_host_is_unresolved() { - let connections: serde_json::Value = serde_json::json!([ - { "Name": "podman-machine-default", "IsMachine": true, "Default": true, - "URI": "ssh://core@127.0.0.1:4444/run/user/1000/podman/podman.sock" } - ]); - // A CONTAINER_HOST that matches no machine connection is unresolved, - // never silently mapped to the default machine. - assert_eq!( - resolve_active_podman_machine(Some("tcp://192.0.2.10:2375"), Some(&connections)), - ActiveMachine::UnresolvedExplicit - ); - } - - #[test] - fn resolve_active_podman_machine_defaults_without_host() { - let connections: serde_json::Value = serde_json::json!([ - { "Name": "podman-machine-default", "IsMachine": true, "Default": true, - "URI": "ssh://core@127.0.0.1:4444/run/user/1000/podman/podman.sock" } - ]); - assert_eq!( - resolve_active_podman_machine(None, Some(&connections)), - ActiveMachine::Default(Some("podman-machine-default".to_string())) - ); - } - - #[test] - fn podman_connection_name_for_uri_ignores_non_machine_matches() { - let connections: serde_json::Value = serde_json::json!([ - { "Name": "remote", "IsMachine": false, "Default": false, - "URI": "tcp://192.0.2.10:2375" } - ]); - // A URI match against a non-machine connection does not map to a local - // machine socket. - assert_eq!( - podman_connection_name_for_uri(&connections, "tcp://192.0.2.10:2375"), - None - ); - } - - #[test] - fn unix_url_socket_path_parses_unix_urls() { - assert_eq!( - unix_url_socket_path("unix:///run/user/1000/podman/podman.sock"), - Some(PathBuf::from("/run/user/1000/podman/podman.sock")) - ); - // Non-unix schemes and empty paths are not sockets. - assert_eq!(unix_url_socket_path("ssh://core@127.0.0.1:22/x"), None); - assert_eq!(unix_url_socket_path("tcp://127.0.0.1:2375"), None); - assert_eq!(unix_url_socket_path("unix://"), None); - } - - #[test] - #[allow(unsafe_code)] // std::env::set_var/remove_var require unsafe in Rust 2024 - fn explicit_unix_container_host_honors_scheme_and_precedence() { - fn set(key: &str, value: Option<&str>) { - unsafe { - match value { - Some(v) => std::env::set_var(key, v), - None => std::env::remove_var(key), - } - } - } - - let original_host = std::env::var("CONTAINER_HOST").ok(); - let original_connection = std::env::var("CONTAINER_CONNECTION").ok(); - - // A unix:// CONTAINER_HOST with no CONTAINER_CONNECTION is used directly. - set("CONTAINER_CONNECTION", None); - set("CONTAINER_HOST", Some("unix:///tmp/podman/custom.sock")); - assert_eq!( - explicit_unix_container_host(), - Some(PathBuf::from("/tmp/podman/custom.sock")) - ); - - // CONTAINER_CONNECTION outranks CONTAINER_HOST. - set("CONTAINER_CONNECTION", Some("work")); - assert_eq!(explicit_unix_container_host(), None); - - // A non-unix CONTAINER_HOST is not a direct socket. - set("CONTAINER_CONNECTION", None); - set( - "CONTAINER_HOST", - Some("ssh://core@127.0.0.1:5555/run/podman.sock"), - ); - assert_eq!(explicit_unix_container_host(), None); - - // Nothing set. - set("CONTAINER_HOST", None); - assert_eq!(explicit_unix_container_host(), None); - - set("CONTAINER_HOST", original_host.as_deref()); - set("CONTAINER_CONNECTION", original_connection.as_deref()); - } - - #[test] - fn parse_default_podman_connection_picks_default_machine() { - let connections: serde_json::Value = serde_json::json!([ - { "Name": "podman-machine-default", "IsMachine": true, "Default": true }, - { "Name": "podman-machine-default-root", "IsMachine": true, "Default": false } - ]); - assert_eq!( - parse_default_podman_connection(&connections), - Some("podman-machine-default".to_string()) - ); - } - - #[test] - fn parse_default_podman_connection_ignores_non_machine_and_missing_default() { - // Default connection that is not a machine is ignored. - let non_machine: serde_json::Value = serde_json::json!([ - { "Name": "remote-host", "IsMachine": false, "Default": true } - ]); - assert_eq!(parse_default_podman_connection(&non_machine), None); - - // No default at all. - let no_default: serde_json::Value = serde_json::json!([ - { "Name": "work", "IsMachine": true, "Default": false } - ]); - assert_eq!(parse_default_podman_connection(&no_default), None); - } } diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index f3279d8105..c8be114ebd 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -437,9 +437,9 @@ pub const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = "/spiffe-workloa /// /// The resulting path is `$XDG_STATE_HOME/openshell/[/]//sandbox.jwt`. /// -/// `driver_subdir` is driver-specific, e.g. `"docker-sandbox-tokens"` or -/// `"podman-sandbox-tokens"`. When `namespace` is `Some`, it is appended as -/// an additional path component (with `/` and `\` replaced by `-`). +/// `driver_subdir` is driver-specific. When `namespace` is `Some`, it is +/// appended as an additional path component (with `/` and `\` replaced by +/// `-`). /// /// # Errors /// Returns an error if the XDG state directory cannot be resolved. @@ -472,7 +472,7 @@ pub fn sandbox_log_level(sandbox: &DriverSandbox, default_level: &str) -> String } // --------------------------------------------------------------------------- -// Supervisor image helpers (shared by Docker and Podman drivers) +// Supervisor image helpers shared by container-backed drivers // --------------------------------------------------------------------------- /// Return the tag portion of a supervisor image reference, or `None` if the @@ -507,7 +507,7 @@ pub fn supervisor_image_should_refresh(image: &str) -> bool { } // --------------------------------------------------------------------------- -// Supervisor binary extraction helpers (shared by Docker and Podman drivers) +// Supervisor binary extraction helpers shared by container-backed drivers // --------------------------------------------------------------------------- #[cfg(feature = "driver-extraction")] @@ -584,8 +584,7 @@ pub fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> Result<(), /// Return the host-side cache path for an extracted supervisor binary. /// /// The path is `$XDG_DATA_HOME/openshell///openshell-sandbox`. -/// `driver_subdir` distinguishes caches across drivers (e.g. `"docker-supervisor"`, -/// `"podman-supervisor"`). +/// `driver_subdir` distinguishes caches across drivers. pub fn supervisor_cache_path(driver_subdir: &str, digest: &str) -> Result { let base = crate::paths::xdg_data_dir() .map_err(|err| format!("failed to resolve XDG data dir: {err}"))?; diff --git a/crates/openshell-core/src/operator_namespace_allowlist.rs b/crates/openshell-core/src/dynamic_string_allowlist.rs similarity index 84% rename from crates/openshell-core/src/operator_namespace_allowlist.rs rename to crates/openshell-core/src/dynamic_string_allowlist.rs index c8f0f7f3de..1a2968188f 100644 --- a/crates/openshell-core/src/operator_namespace_allowlist.rs +++ b/crates/openshell-core/src/dynamic_string_allowlist.rs @@ -4,16 +4,13 @@ use std::collections::BTreeSet; use std::sync::{Arc, RwLock}; -/// Thread-safe dynamic allowlist of Kubernetes operator-mode namespaces. -/// -/// This type lives in the public core API because both the Kubernetes driver -/// and gateway authentication boundary consume it. +/// Thread-safe dynamic allowlist of strings shared across component boundaries. #[derive(Debug, Clone)] -pub struct OperatorNamespaceAllowlist { +pub struct DynamicStringAllowlist { inner: Arc>>, } -impl OperatorNamespaceAllowlist { +impl DynamicStringAllowlist { fn read_guard(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { self.inner .read() @@ -71,7 +68,7 @@ impl OperatorNamespaceAllowlist { } } -impl Default for OperatorNamespaceAllowlist { +impl Default for DynamicStringAllowlist { fn default() -> Self { Self::new() } diff --git a/crates/openshell-core/src/error.rs b/crates/openshell-core/src/error.rs index 8c23e30198..145106012d 100644 --- a/crates/openshell-core/src/error.rs +++ b/crates/openshell-core/src/error.rs @@ -106,8 +106,8 @@ impl Error { /// Error type shared by all compute driver implementations. /// -/// Both the Podman and Kubernetes drivers map their backend-specific -/// errors into these variants before crossing crate boundaries. +/// Drivers map backend-specific errors into these variants before crossing +/// crate boundaries. #[derive(Debug, Error)] pub enum ComputeDriverError { /// The requested sandbox already exists. diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 315bb319bb..f4170002c7 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -16,6 +16,7 @@ pub mod container_paths; pub mod denial; pub mod driver_mounts; pub mod driver_utils; +pub mod dynamic_string_allowlist; pub mod endpoint_path; pub mod error; #[cfg(unix)] @@ -28,12 +29,12 @@ pub mod host_pattern; pub mod image; pub mod inference; pub mod jwt; +pub mod local_api_socket; pub mod metadata; pub mod middleware; pub mod net; #[cfg(feature = "oauth")] pub mod oauth; -pub mod operator_namespace_allowlist; pub mod paths; pub mod policy; pub mod progress; @@ -44,22 +45,21 @@ pub mod provider_credentials; pub mod sandbox_env; pub mod secrets; pub mod settings; -pub mod spiffe; pub mod telemetry; pub mod time; pub mod transport_errors; pub use config::{ - ComputeDriverKind, Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, - GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, - GatewayInterceptorPhaseConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, - MtlsAuthConfig, OidcConfig, PolicyValidationFailureMode, TlsConfig, + Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, GatewayInterceptorBindingPolicy, + GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayInterceptorPhaseConfig, + GatewayJwtConfig, GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, + PolicyValidationFailureMode, TlsConfig, }; +pub use dynamic_string_allowlist::DynamicStringAllowlist; pub use error::{ComputeDriverError, Error, Result}; pub use metadata::{ GetResourceVersion, ObjectId, ObjectLabels, ObjectName, ObjectWorkspace, SetResourceVersion, }; -pub use operator_namespace_allowlist::OperatorNamespaceAllowlist; /// Build version string derived from git metadata. /// diff --git a/crates/openshell-core/src/local_api_socket.rs b/crates/openshell-core/src/local_api_socket.rs new file mode 100644 index 0000000000..6804ae513a --- /dev/null +++ b/crates/openshell-core/src/local_api_socket.rs @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Generic discovery and probing for local HTTP APIs over Unix sockets. + +use std::path::{Path, PathBuf}; + +/// Return the first candidate whose HTTP ping response is accepted. +#[must_use] +pub fn first_responsive_socket( + candidates: &[PathBuf], + accepts_response: impl Fn(&[u8]) -> bool, +) -> Option { + candidates + .iter() + .find(|path| socket_responds(path, &accepts_response)) + .cloned() +} + +/// Return whether a byte slice contains another, ignoring ASCII case. +#[must_use] +pub fn contains_ascii(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window.eq_ignore_ascii_case(needle)) +} + +/// Return whether an HTTP response starts with a successful status line. +#[must_use] +pub fn http_response_is_success(response: &[u8]) -> bool { + response.starts_with(b"HTTP/1.1 200") || response.starts_with(b"HTTP/1.0 200") +} + +#[cfg(unix)] +fn socket_responds(path: &Path, accepts_response: &impl Fn(&[u8]) -> bool) -> bool { + use std::io::{Read as _, Write as _}; + use std::os::unix::fs::FileTypeExt as _; + use std::time::Duration; + + const PROBE_TIMEOUT: Duration = Duration::from_secs(1); + const PING_REQUEST: &[u8] = + b"GET /_ping HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; + + if !path + .metadata() + .is_ok_and(|metadata| metadata.file_type().is_socket()) + { + return false; + } + let Ok(mut stream) = std::os::unix::net::UnixStream::connect(path) else { + return false; + }; + if stream.set_read_timeout(Some(PROBE_TIMEOUT)).is_err() + || stream.set_write_timeout(Some(PROBE_TIMEOUT)).is_err() + || stream.write_all(PING_REQUEST).is_err() + { + return false; + } + + let mut response = [0_u8; 512]; + let mut total = 0; + while total < response.len() { + let Ok(read) = stream.read(&mut response[total..]) else { + return false; + }; + if read == 0 { + break; + } + total += read; + if contains_ascii(&response[..total], b"\r\n\r\n") { + break; + } + } + total > 0 && accepts_response(&response[..total]) +} + +#[cfg(not(unix))] +fn socket_responds(path: &Path, accepts_response: &impl Fn(&[u8]) -> bool) -> bool { + let _ = (path, accepts_response); + false +} diff --git a/crates/openshell-core/src/settings.rs b/crates/openshell-core/src/settings.rs index 625ba4c856..b7bf72c3d3 100644 --- a/crates/openshell-core/src/settings.rs +++ b/crates/openshell-core/src/settings.rs @@ -65,7 +65,7 @@ impl RegisteredSetting { /// /// 1. Add a [`RegisteredSetting`] entry to this array with the key name and /// [`SettingValueKind`]. -/// 2. Recompile `openshell-server` (gateway) and `openshell-sandbox` +/// 2. Recompile `openshell-gateway` and `openshell-sandbox` /// (supervisor). No database migration is needed -- new keys are stored in /// the existing settings JSON blob. /// 3. Add sandbox-side consumption in `openshell-sandbox` to read and act on diff --git a/crates/openshell-core/src/telemetry.rs b/crates/openshell-core/src/telemetry.rs index ad15989944..f092f5f8aa 100644 --- a/crates/openshell-core/src/telemetry.rs +++ b/crates/openshell-core/src/telemetry.rs @@ -159,51 +159,20 @@ impl SandboxTemplateSource { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TelemetryComputeDriver { - Docker, - Kubernetes, - Podman, - Vm, - Mxc, - Unknown, -} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TelemetryComputeDriver(String); impl TelemetryComputeDriver { #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::Docker => "docker", - Self::Kubernetes => "kubernetes", - Self::Podman => "podman", - Self::Vm => "vm", - Self::Mxc => "mxc", - Self::Unknown => "unknown", - } + pub fn as_str(&self) -> &str { + &self.0 } #[must_use] pub fn from_raw(raw: &str) -> Self { - match raw.trim().to_ascii_lowercase().as_str() { - "docker" => Self::Docker, - "k8s" | "kubernetes" => Self::Kubernetes, - "podman" => Self::Podman, - "vm" => Self::Vm, - "mxc" => Self::Mxc, - _ => Self::Unknown, - } - } - - #[must_use] - pub const fn from_driver_kind(driver_kind: Option) -> Self { - match driver_kind { - Some(crate::ComputeDriverKind::Docker) => Self::Docker, - Some(crate::ComputeDriverKind::Kubernetes) => Self::Kubernetes, - Some(crate::ComputeDriverKind::Podman) => Self::Podman, - Some(crate::ComputeDriverKind::Vm) => Self::Vm, - Some(crate::ComputeDriverKind::Mxc) => Self::Mxc, - None => Self::Unknown, - } + let name = crate::config::normalize_compute_driver_name(raw) + .unwrap_or_else(|_| "unknown".to_string()); + Self(name) } } @@ -688,27 +657,21 @@ mod tests { } #[test] - fn compute_driver_values_are_sanitized() { - assert_eq!( - TelemetryComputeDriver::from_raw("docker").as_str(), - "docker" - ); - assert_eq!( - TelemetryComputeDriver::from_raw("k8s").as_str(), - "kubernetes" - ); + fn compute_driver_values_are_normalized_without_enumerating_backends() { + assert_eq!(TelemetryComputeDriver::from_raw("alpha").as_str(), "alpha"); assert_eq!( - TelemetryComputeDriver::from_raw("KUBERNETES").as_str(), - "kubernetes" + TelemetryComputeDriver::from_raw(" Alpha ").as_str(), + "alpha" ); - assert_eq!(TelemetryComputeDriver::from_raw("vm").as_str(), "vm"); assert_eq!( - TelemetryComputeDriver::from_raw("podman").as_str(), - "podman" + TelemetryComputeDriver::from_raw("CUSTOM_BACKEND").as_str(), + "custom_backend" ); + assert_eq!(TelemetryComputeDriver::from_raw("beta").as_str(), "beta"); + assert_eq!(TelemetryComputeDriver::from_raw("gamma").as_str(), "gamma"); assert_eq!( TelemetryComputeDriver::from_raw("private-driver").as_str(), - "unknown" + "private-driver" ); } @@ -797,7 +760,7 @@ mod disabled_tests { 1, false, SandboxTemplateSource::Default, - TelemetryComputeDriver::Docker, + TelemetryComputeDriver::from_raw("test-driver"), ); emit_policy_decision( PolicyDecisionOperation::Approve, diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 3941819c61..6a46af4bef 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5,8 +5,6 @@ #![allow(clippy::result_large_err)] -pub mod otel_tracing; - use bollard::Docker; use bollard::errors::Error as BollardError; use bollard::models::{ @@ -21,9 +19,7 @@ use bollard::query_parameters::{ }; use bytes::Bytes; use futures::{Stream, StreamExt}; -use openshell_core::config::{ - DEFAULT_DOCKER_NETWORK_NAME, DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS, -}; +use openshell_core::config::{DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, @@ -56,22 +52,18 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; -use openshell_core::{Config, Error, Result as CoreResult}; -use opentelemetry::trace::TraceContextExt as _; +use openshell_core::{Error, Result as CoreResult}; use std::collections::{HashMap, HashSet}; -use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; -use std::task::{Context, Poll}; use std::time::Duration; use tokio::sync::{Mutex, broadcast, mpsc}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; -use tracing::{Instrument as _, debug, info, warn}; -use tracing_opentelemetry::OpenTelemetrySpanExt as _; +use tracing::{debug, info, warn}; use url::Url; const WATCH_BUFFER: usize = 128; @@ -88,28 +80,6 @@ const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; const DOCKER_NETWORK_DRIVER: &str = "bridge"; -fn provisioning_span( - parent: &opentelemetry::Context, - sandbox: &DriverSandbox, - image_ref: &str, -) -> tracing::Span { - let span = tracing::info_span!( - parent: None, - "docker.provision", - otel.name = "docker.provision", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox.id, - sandbox.name = %sandbox.name, - image.ref = %image_ref, - ); - let parent_span_context = parent.span().span_context().clone(); - if parent_span_context.is_valid() { - let parent = opentelemetry::Context::new().with_remote_span_context(parent_span_context); - let _ = span.set_parent(parent); - } - span -} - /// Gateway-local configuration for the Docker compute driver. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] @@ -415,125 +385,45 @@ fn default_true() -> bool { type WatchStream = Pin> + Send + 'static>>; -struct TracedWatchStream { - inner: WatchStream, - span: tracing::Span, - finished: bool, -} - -impl Stream for TracedWatchStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let span = self.span.clone(); - let _entered = span.enter(); - let result = self.inner.as_mut().poll_next(cx); - if !self.finished { - match &result { - Poll::Ready(Some(Err(status))) => { - openshell_otel::mark_error(&self.span); - self.span - .record("rpc.grpc.status_code", status.code() as i32); - self.finished = true; - } - Poll::Ready(None) => { - self.span - .record("rpc.grpc.status_code", tonic::Code::Ok as i32); - self.finished = true; - } - Poll::Pending | Poll::Ready(Some(Ok(_))) => {} - } - } - result - } -} - -impl Drop for TracedWatchStream { - fn drop(&mut self) { - if !self.finished { - openshell_otel::mark_error(&self.span); - self.span - .record("rpc.grpc.status_code", tonic::Code::Cancelled as i32); - } - } -} - -/// Compute-driver service wrapper that preserves the standalone RPC trace -/// boundary while Docker runs in the gateway process. -#[derive(Clone)] -pub struct ComputeDriverService { - driver: DockerComputeDriver, - trace_in_process_rpc: bool, -} - -impl ComputeDriverService { - #[must_use] - pub fn new(driver: DockerComputeDriver) -> Self { - Self { - driver, - trace_in_process_rpc: false, - } +/// Return the first responsive local Docker API socket. +#[must_use] +pub fn detect_socket() -> Option { + let mut candidates = Vec::new(); + if let Ok(host) = std::env::var("DOCKER_HOST") + && let Some(path) = host.trim().strip_prefix("unix://") + && !path.is_empty() + { + candidates.push(PathBuf::from(path)); } - - #[must_use] - pub fn new_in_process(driver: DockerComputeDriver) -> Self { - Self { - driver, - trace_in_process_rpc: true, - } + candidates.push(PathBuf::from("/var/run/docker.sock")); + if let Some(home) = std::env::var_os("HOME") { + candidates.push(PathBuf::from(home).join(".docker/run/docker.sock")); } - - fn in_process_rpc_span( - &self, - operation: &'static str, - method: &'static str, - ) -> Option { - self.trace_in_process_rpc.then(|| { - tracing::info_span!( - target: "openshell_driver_docker::otel_tracing", - "driver_rpc", - otel.name = operation, - otel.kind = "server", - otel.status_code = tracing::field::Empty, - rpc.system = "grpc", - rpc.service = "openshell.compute.v1.ComputeDriver", - rpc.method = method, - rpc.grpc.status_code = tracing::field::Empty, - ) - }) + if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { + candidates.push(PathBuf::from(runtime_dir).join("docker.sock")); } + openshell_core::local_api_socket::first_responsive_socket(&candidates, |response| { + openshell_core::local_api_socket::http_response_is_success(response) + && openshell_core::local_api_socket::contains_ascii(response, b"Api-Version:") + && !openshell_core::local_api_socket::contains_ascii(response, b"Libpod-Api-Version:") + }) +} - async fn trace_rpc( - &self, - operation: &'static str, - method: &'static str, - future: impl Future>, - ) -> Result { - use tracing::Instrument as _; - - let Some(span) = self.in_process_rpc_span(operation, method) else { - return future.await; - }; - let result = future.instrument(span.clone()).await; - match &result { - Ok(_) => { - span.record("rpc.grpc.status_code", tonic::Code::Ok as i32); - } - Err(status) => { - openshell_otel::mark_error(&span); - span.record("rpc.grpc.status_code", status.code() as i32); - } - } - result - } +#[must_use] +pub fn is_available() -> bool { + detect_socket().is_some() } impl DockerComputeDriver { - pub async fn new(config: &Config, docker_config: &DockerComputeConfig) -> CoreResult { + pub async fn new( + gateway_bind_address: SocketAddr, + gateway_log_level: &str, + docker_config: &DockerComputeConfig, + ) -> CoreResult { let socket_path = docker_config .socket_path .clone() - .or_else(openshell_core::config::detect_docker_socket) + .or_else(detect_socket) .unwrap_or_else(|| PathBuf::from("/var/run/docker.sock")); let socket_path_str = socket_path.to_str().ok_or_else(|| { Error::config(format!( @@ -559,7 +449,7 @@ impl DockerComputeDriver { let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); let allow_all_default_gpu = docker_info_reports_wsl2(&info); validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; - let gateway_port = config.bind_address.port(); + let gateway_port = gateway_bind_address.port(); if gateway_port == 0 { return Err(Error::config( "docker compute driver requires a fixed non-zero gateway bind port", @@ -571,7 +461,7 @@ impl DockerComputeDriver { let gateway_route = docker_gateway_route(&info, bridge_gateway_ip, gateway_port, host_gateway_ip); let gateway_callback_bind_address = - docker_gateway_callback_bind_address(&gateway_route, config.bind_address); + docker_gateway_callback_bind_address(&gateway_route, gateway_bind_address); let mut docker_config = docker_config.clone(); if docker_config.grpc_endpoint.trim().is_empty() { let scheme = if docker_guest_tls_configured(&docker_config) { @@ -603,7 +493,7 @@ impl DockerComputeDriver { gateway_callback_bind_address, ssh_socket_path: docker_config.ssh_socket_path.clone(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, - log_level: config.log_level.clone(), + log_level: gateway_log_level.to_string(), supervisor_bin, guest_tls, daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), @@ -897,7 +787,7 @@ impl DockerComputeDriver { &sandbox.id, "Scheduled", format!("Docker sandbox accepted for image \"{image}\""), - HashMap::from([("image_ref".to_string(), image.clone())]), + HashMap::from([("image_ref".to_string(), image)]), ); self.publish_sandbox_snapshot(pending_sandbox_snapshot( sandbox, @@ -909,14 +799,9 @@ impl DockerComputeDriver { let driver = self.clone(); let sandbox_for_task = sandbox.clone(); let sandbox_id = sandbox.id.clone(); - let parent = tracing::Span::current().context(); - let provisioning_span = provisioning_span(&parent, sandbox, &image); - let task = tokio::spawn( - async move { - driver.provision_sandbox(sandbox_for_task).await; - } - .instrument(provisioning_span), - ); + let task = tokio::spawn(async move { + driver.provision_sandbox(sandbox_for_task).await; + }); let mut pending = self.pending.lock().await; if let Some(record) = pending.get_mut(&sandbox_id) { @@ -939,42 +824,20 @@ impl DockerComputeDriver { } } - #[tracing::instrument( - name = "docker.provision_sandbox", - skip(self, sandbox), - fields( - otel.name = "docker.provision_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox.id, - sandbox.name = %sandbox.name, - ) - )] async fn provision_sandbox_inner( &self, sandbox: &DriverSandbox, ) -> Result<(), DockerProvisioningFailure> { - let span_status = openshell_otel::ErrorStatusGuard::current(); let validated = Self::validated_sandbox(sandbox, &self.config).map_err(|status| { DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; let template = validated.template; - let image = async { - openshell_otel::record_error_result( - self.ensure_image_available(&sandbox.id, &template.image) - .await - .map_err(|status| { - DockerProvisioningFailure::new("ImagePullFailed", status.message()) - }), - ) - } - .instrument(tracing::info_span!( - "docker.prepare_image", - otel.name = "docker.prepare_image", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox.id, - image.ref = %template.image, - )) - .await?; + let image = self + .ensure_image_available(&sandbox.id, &template.image) + .await + .map_err(|status| { + DockerProvisioningFailure::new("ImagePullFailed", status.message()) + })?; let token_file_created = write_sandbox_token_file(sandbox, &self.config) .await .map_err(|status| { @@ -1008,37 +871,25 @@ impl DockerComputeDriver { } DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; - async { - openshell_otel::record_error_result( - self.docker - .create_container( - Some( - CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(), - ), - create_body, - ) - .await - .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::from_status( - "ContainerCreateFailed", - create_status_from_docker_error("create docker sandbox container", err), - ) - }), + self.docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(container_name.as_str()) + .build(), + ), + create_body, ) - } - .instrument(tracing::info_span!( - "docker.create_container", - otel.name = "docker.create_container", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox.id, - container.name = %container_name, - )) - .await?; + .await + .map_err(|err| { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + DockerProvisioningFailure::from_status( + "ContainerCreateFailed", + create_status_from_docker_error("create docker sandbox container", err), + ) + })?; self.publish_docker_progress( &sandbox.id, "Created", @@ -1046,20 +897,7 @@ impl DockerComputeDriver { HashMap::from([("container_name".to_string(), container_name.clone())]), ); - let start_result = async { - openshell_otel::record_error_result( - self.docker.start_container(&container_name, None).await, - ) - } - .instrument(tracing::info_span!( - "docker.start_container", - otel.name = "docker.start_container", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox.id, - container.name = %container_name, - )) - .await; - if let Err(err) = start_result { + if let Err(err) = self.docker.start_container(&container_name, None).await { let cleanup = self .docker .remove_container( @@ -1100,7 +938,7 @@ impl DockerComputeDriver { ); } - span_status.finish(Ok(())) + Ok(()) } async fn delete_sandbox_inner( @@ -1214,29 +1052,17 @@ impl DockerComputeDriver { /// Returns `Ok(true)` when a container existed and was started (or was /// already running), `Ok(false)` when no managed container is found for /// the sandbox, and `Err(...)` for any Docker failure. - #[tracing::instrument( - name = "docker.start_sandbox", - skip(self), - fields( - otel.name = "docker.start_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, - sandbox.name = %sandbox_name, - ) - )] pub async fn start_sandbox( &self, sandbox_id: &str, sandbox_name: &str, ) -> Result { - let span_status = openshell_otel::ErrorStatusGuard::current(); - require_sandbox_identifier(sandbox_id, sandbox_name)?; self.lifecycle_event_fences.begin_start(sandbox_id); let result = self .start_sandbox_with_lifecycle_fence(sandbox_id, sandbox_name) .await; self.lifecycle_event_fences.finish_start(sandbox_id); - span_status.finish(result) + result } async fn start_sandbox_with_lifecycle_fence( @@ -1737,188 +1563,12 @@ impl DockerComputeDriver { } } -#[tonic::async_trait] -impl ComputeDriver for ComputeDriverService { - type WatchSandboxesStream = WatchStream; - - async fn authenticate_sandbox( - &self, - request: Request, - ) -> Result, Status> - { - self.trace_rpc( - "driver.authenticate_sandbox", - "authenticate_sandbox", - ComputeDriver::authenticate_sandbox(&self.driver, request), - ) - .await - } - - async fn get_capabilities( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.get_capabilities", - "get_capabilities", - ComputeDriver::get_capabilities(&self.driver, request), - ) - .await - } - - async fn get_gateway_listener_requirements( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.get_gateway_listener_requirements", - "get_gateway_listener_requirements", - ComputeDriver::get_gateway_listener_requirements(&self.driver, request), - ) - .await - } - - async fn validate_sandbox_create( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.validate_sandbox_create", - "validate_sandbox_create", - ComputeDriver::validate_sandbox_create(&self.driver, request), - ) - .await - } - - async fn get_sandbox( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.get_sandbox", - "get_sandbox", - ComputeDriver::get_sandbox(&self.driver, request), - ) - .await - } - - async fn list_sandboxes( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.list_sandboxes", - "list_sandboxes", - ComputeDriver::list_sandboxes(&self.driver, request), - ) - .await - } - - async fn create_sandbox( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.create_sandbox", - "create_sandbox", - ComputeDriver::create_sandbox(&self.driver, request), - ) - .await - } - - async fn stop_sandbox( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.stop_sandbox", - "stop_sandbox", - ComputeDriver::stop_sandbox(&self.driver, request), - ) - .await - } - - async fn start_sandbox( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.start_sandbox", - "start_sandbox", - ComputeDriver::start_sandbox(&self.driver, request), - ) - .await - } - - async fn delete_sandbox( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.delete_sandbox", - "delete_sandbox", - ComputeDriver::delete_sandbox(&self.driver, request), - ) - .await - } - - async fn watch_sandboxes( - &self, - request: Request, - ) -> Result, Status> { - use tracing::Instrument as _; - - let create_stream = ComputeDriver::watch_sandboxes(&self.driver, request); - let Some(span) = self.in_process_rpc_span("driver.watch_sandboxes", "watch_sandboxes") - else { - return create_stream.await; - }; - match create_stream.instrument(span.clone()).await { - Ok(response) => Ok(Response::new(Box::pin(TracedWatchStream { - inner: response.into_inner(), - span, - finished: false, - }))), - Err(status) => { - openshell_otel::mark_error(&span); - span.record("rpc.grpc.status_code", status.code() as i32); - Err(status) - } - } - } - - async fn ensure_workspace( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.ensure_workspace", - "ensure_workspace", - ComputeDriver::ensure_workspace(&self.driver, request), - ) - .await - } - - async fn delete_workspace( - &self, - request: Request, - ) -> Result, Status> { - self.trace_rpc( - "driver.delete_workspace", - "delete_workspace", - ComputeDriver::delete_workspace(&self.driver, request), - ) - .await - } -} - #[tonic::async_trait] impl ComputeDriver for DockerComputeDriver { async fn authenticate_sandbox( &self, _request: Request, - ) -> Result, Status> - { + ) -> Result, Status> { Err(Status::unimplemented( "docker does not authenticate sandbox credentials", )) @@ -2008,44 +1658,22 @@ impl ComputeDriver for DockerComputeDriver { })) } - #[tracing::instrument( - name = "docker.schedule_sandbox", - skip(self, request), - fields( - otel.name = "docker.schedule_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %request.get_ref().sandbox.as_ref().map_or("", |sandbox| sandbox.id.as_str()), - sandbox.name = %request.get_ref().sandbox.as_ref().map_or("", |sandbox| sandbox.name.as_str()), - ) - )] async fn create_sandbox( &self, request: Request, ) -> Result, Status> { - let span_status = openshell_otel::ErrorStatusGuard::current(); let sandbox = request .into_inner() .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; self.create_sandbox_inner(&sandbox).await?; - span_status.finish(Ok(Response::new(CreateSandboxResponse {}))) + Ok(Response::new(CreateSandboxResponse {})) } - #[tracing::instrument( - name = "docker.stop_sandbox", - skip(self, request), - fields( - otel.name = "docker.stop_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %request.get_ref().sandbox_id, - sandbox.name = %request.get_ref().sandbox_name, - ) - )] async fn stop_sandbox( &self, request: Request, ) -> Result, Status> { - let span_status = openshell_otel::ErrorStatusGuard::current(); let request = request.into_inner(); require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; @@ -2053,7 +1681,7 @@ impl ComputeDriver for DockerComputeDriver { .await?; self.publish_container_snapshot(&request.sandbox_id, &request.sandbox_name) .await?; - span_status.finish(Ok(Response::new(StopSandboxResponse {}))) + Ok(Response::new(StopSandboxResponse {})) } async fn start_sandbox( @@ -2061,6 +1689,7 @@ impl ComputeDriver for DockerComputeDriver { request: Request, ) -> Result, Status> { let request = request.into_inner(); + require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; if !Self::start_sandbox(self, &request.sandbox_id, &request.sandbox_name).await? { return Err(Status::not_found("sandbox not found")); } @@ -2069,21 +1698,10 @@ impl ComputeDriver for DockerComputeDriver { Ok(Response::new(StartSandboxResponse {})) } - #[tracing::instrument( - name = "docker.delete_sandbox", - skip(self, request), - fields( - otel.name = "docker.delete_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %request.get_ref().sandbox_id, - sandbox.name = %request.get_ref().sandbox_name, - ) - )] async fn delete_sandbox( &self, request: Request, ) -> Result, Status> { - let span_status = openshell_otel::ErrorStatusGuard::current(); let request = request.into_inner(); require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; @@ -2102,7 +1720,7 @@ impl ComputeDriver for DockerComputeDriver { }); } - span_status.finish(Ok(Response::new(DeleteSandboxResponse { deleted }))) + Ok(Response::new(DeleteSandboxResponse { deleted })) } async fn watch_sandboxes( @@ -4140,3 +3758,4 @@ fn internal_status(operation: &str, err: BollardError) -> Status { #[cfg(test)] mod tests; +pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs index 2c6dbfd4c1..7c4b5b1cb4 100644 --- a/crates/openshell-driver-docker/src/main.rs +++ b/crates/openshell-driver-docker/src/main.rs @@ -7,7 +7,7 @@ use std::path::PathBuf; use clap::Parser; use miette::{IntoDiagnostic, Result}; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_core::{Config, VERSION}; +use openshell_core::VERSION; use openshell_driver_docker::otel_tracing::compute_driver_rpc_layer; use openshell_driver_docker::{ComputeDriverService, DockerComputeConfig, DockerComputeDriver}; use tracing::info; @@ -67,8 +67,7 @@ async fn main() -> Result<()> { let config_source = std::fs::read_to_string(&args.config).into_diagnostic()?; let docker_config: DockerComputeConfig = toml::from_str(&config_source).into_diagnostic()?; - let gateway_config = Config::new(None).with_bind_address(args.gateway_bind); - let driver = DockerComputeDriver::new(&gateway_config, &docker_config) + let driver = DockerComputeDriver::new(args.gateway_bind, &args.log_level, &docker_config) .await .into_diagnostic()?; diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index aadcb1342a..805c0314b0 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -pub use openshell_core::OperatorNamespaceAllowlist; +pub use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist; use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::BTreeMap; diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index edc0a3e97e..28d3c77a7d 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -14,4 +14,4 @@ pub use config::{ }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; -pub use openshell_core::OperatorNamespaceAllowlist; +pub use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist; diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index ccd4413b3f..e53ddc9f3f 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -457,11 +457,11 @@ matter compared to cluster or rootful runtimes: ## Implementation References -- Gateway integration: `crates/openshell-server/src/compute/mod.rs` - (`new_podman` and `PodmanComputeDriver` wiring). -- Server configuration: `crates/openshell-server/src/lib.rs` - (`ComputeDriverKind::Podman` builds `PodmanComputeConfig` including - `sandbox_ssh_socket_path` from gateway `Config`). +- Gateway integration: `crates/openshell-gateway/src/lib.rs` registers the + driver factory and constructs `PodmanComputeConfig` from the generic server + build context. +- Server configuration: `crates/openshell-server/src/lib.rs` exposes the + backend-agnostic registry and factory context. - Gateway relay path: `openshell-core` `Config::sandbox_ssh_socket_path` in `crates/openshell-core/src/config.rs`. - SSRF mitigation: `crates/openshell-core/src/net.rs`, diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 8f7c0d32f6..f3acd1ebf8 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -311,11 +311,42 @@ fn podman_gpu_selection_error(err: CdiGpuSelectionError) -> ComputeDriverError { ComputeDriverError::Precondition(err.to_string()) } +/// Return the first responsive local Podman API socket. +#[must_use] +pub fn detect_socket() -> Option { + let mut candidates = Vec::new(); + if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") + && !path.trim().is_empty() + { + candidates.push(PathBuf::from(path)); + } + if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { + candidates.push(PathBuf::from(runtime_dir).join("podman/podman.sock")); + } + #[cfg(target_os = "linux")] + candidates.push(PathBuf::from(format!( + "/run/user/{}/podman/podman.sock", + rustix::process::geteuid().as_raw() + ))); + if let Some(home) = std::env::var_os("HOME") { + candidates + .push(PathBuf::from(home).join(".local/share/containers/podman/machine/podman.sock")); + } + openshell_core::local_api_socket::first_responsive_socket(&candidates, |response| { + openshell_core::local_api_socket::http_response_is_success(response) + && openshell_core::local_api_socket::contains_ascii(response, b"Libpod-Api-Version:") + }) +} + +#[must_use] +pub fn is_available() -> bool { + detect_socket().is_some() +} + /// Resolve the socket to connect to: explicit configuration wins, otherwise /// fall back to `detect`. Returns an error if neither resolves. /// -/// Takes `detect` as a parameter (rather than calling -/// [`openshell_core::config::detect_podman_socket`] directly) so tests can +/// Takes `detect` as a parameter so tests can /// exercise the precedence deterministically, without touching real /// environment variables or the filesystem. fn resolve_socket_path( @@ -337,10 +368,7 @@ impl PodmanComputeDriver { const MAX_PING_RETRIES: u32 = 5; const PING_RETRY_DELAY: Duration = Duration::from_secs(2); - let socket_path = resolve_socket_path( - config.socket_path.clone(), - openshell_core::config::detect_podman_socket, - )?; + let socket_path = resolve_socket_path(config.socket_path.clone(), detect_socket)?; config.socket_path = Some(socket_path.clone()); if !socket_path.exists() { diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index 0c94bf72b2..c3903a0067 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -122,15 +122,12 @@ fn deleted_event(sandbox_id: String) -> WatchSandboxesEvent { /// drops (daemon restart, socket error, or clean shutdown), the stream /// terminates with a final error item and stops producing events. /// -/// Callers are responsible for reconnecting by calling [`start_watch`] again. -/// The server's `ComputeRuntime::watch_loop` in `openshell-server` provides -/// this behaviour with a 2-second backoff: when the stream terminates with an -/// error, `watch_loop` sleeps and then calls `watch_sandboxes()` again, which -/// ultimately calls `start_watch()` again and re-syncs state. +/// Callers are responsible for reconnecting by calling [`start_watch`] again +/// and re-synchronizing state. /// /// **Do not add reconnection logic inside this function.** A local reconnect -/// would race with `watch_loop`'s retry and produce duplicate initial-sync -/// events that corrupt the server's sandbox index. +/// would race with the consumer's retry and produce duplicate initial-sync +/// events. pub async fn start_watch( client: PodmanClient, lifecycle_event_fences: LifecycleEventFences, diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index f455beaeb5..4fc9ace415 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -9,7 +9,7 @@ Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) fo ```mermaid flowchart LR subgraph host["Host process"] - gateway["openshell-server
(compute::vm::spawn)"] + gateway["openshell-gateway
(vm::spawn)"] driver["openshell-driver-vm
├── libkrun (VM)
├── gvproxy (net)
└── openshell-sandbox.zst"] gateway <-->|"gRPC over UDS
compute-driver.sock"| driver end @@ -102,7 +102,7 @@ mise run vm:supervisor # if openshell-sandbox.zst is not already presen # 2. Build both binaries with the staged artifacts embedded OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ - cargo build -p openshell-server -p openshell-driver-vm + cargo build -p openshell-gateway -p openshell-driver-vm # 3. macOS only: codesign the driver for Hypervisor.framework codesign \ @@ -291,5 +291,5 @@ the user explicitly overrides it. ## TODOs -- The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` notes in `crates/openshell-server/src/lib.rs` and `crates/openshell-server/src/compute/vm.rs`. +- The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` note in `crates/openshell-gateway/src/vm.rs`. - macOS local builds are codesigned by `tasks/scripts/gateway-vm.sh`; the generated Homebrew formula signs the release tarball driver for local installs. diff --git a/crates/openshell-driver-vm/runtime/README.md b/crates/openshell-driver-vm/runtime/README.md index 11aab67f43..b686874ba2 100644 --- a/crates/openshell-driver-vm/runtime/README.md +++ b/crates/openshell-driver-vm/runtime/README.md @@ -41,7 +41,7 @@ mise run vm:supervisor # Build the gateway and VM driver with embedded runtime artifacts OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ - cargo build -p openshell-server -p openshell-driver-vm + cargo build -p openshell-gateway -p openshell-driver-vm ``` Use `FROM_SOURCE=1 mise run vm:setup` to build the runtime from source instead diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 2de65c3add..bc11a1453c 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -3663,7 +3663,7 @@ async fn connect_local_container_engine() -> Option { return Some(docker); } - let podman_socket = openshell_core::config::detect_podman_socket()?; + let podman_socket = detect_podman_socket()?; if let Ok(docker) = Docker::connect_with_unix(podman_socket.to_str()?, 120, bollard::API_DEFAULT_VERSION) && docker.ping().await.is_ok() @@ -3678,6 +3678,31 @@ async fn connect_local_container_engine() -> Option { None } +fn detect_podman_socket() -> Option { + let mut candidates = Vec::new(); + if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") + && !path.trim().is_empty() + { + candidates.push(PathBuf::from(path)); + } + if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { + candidates.push(PathBuf::from(runtime_dir).join("podman/podman.sock")); + } + #[cfg(target_os = "linux")] + candidates.push(PathBuf::from(format!( + "/run/user/{}/podman/podman.sock", + rustix::process::geteuid().as_raw() + ))); + if let Some(home) = std::env::var_os("HOME") { + candidates + .push(PathBuf::from(home).join(".local/share/containers/podman/machine/podman.sock")); + } + openshell_core::local_api_socket::first_responsive_socket(&candidates, |response| { + openshell_core::local_api_socket::http_response_is_success(response) + && openshell_core::local_api_socket::contains_ascii(response, b"Libpod-Api-Version:") + }) +} + fn is_openshell_local_build_image_ref(image_ref: &str) -> bool { image_ref.starts_with("openshell/sandbox-from:") } diff --git a/crates/openshell-gateway/BUILD.bazel b/crates/openshell-gateway/BUILD.bazel new file mode 100644 index 0000000000..90b464ef15 --- /dev/null +++ b/crates/openshell-gateway/BUILD.bazel @@ -0,0 +1,64 @@ +load("@crates//:defs.bzl", "aliases", "all_crate_deps") +load("@rules_rs//rs:rust_binary.bzl", "rust_binary") +load("@rules_rs//rs:rust_library.bzl", "rust_library") +load("@rules_rs//rs:rust_test.bzl", "rust_test") +load("@rules_rust//rust:defs.bzl", "rustfmt_test") +load("@workspace_version//:version.bzl", "WORKSPACE_VERSION") + +rust_library( + name = "openshell-gateway", + srcs = glob( + ["src/**/*.rs"], + exclude = ["src/main.rs"], + ), + aliases = aliases(), + crate_features = [ + "in-tree-compute-drivers", + "telemetry", + ], + version = WORKSPACE_VERSION, + visibility = ["//visibility:public"], + deps = all_crate_deps(normal = True), +) + +rust_binary( + name = "openshell-gateway-bin", + srcs = ["src/main.rs"], + aliases = aliases(), + binary_name = "openshell-gateway", + version = WORKSPACE_VERSION, + visibility = ["//visibility:public"], + deps = all_crate_deps(normal = True) + [":openshell-gateway"], +) + +rust_test( + name = "openshell-gateway_lib_test", + crate = ":openshell-gateway", + crate_features = [ + "in-tree-compute-drivers", + "telemetry", + ], + deps = all_crate_deps(normal_dev = True), +) + +rust_test( + name = "openshell-gateway_bin_test", + srcs = ["src/main.rs"], + aliases = aliases(), + version = WORKSPACE_VERSION, + deps = all_crate_deps( + normal = True, + normal_dev = True, + ) + [":openshell-gateway"], +) + +rustfmt_test( + name = "rustfmt_test", + targets = [ + ":openshell-gateway", + ":openshell-gateway-bin", + ":openshell-gateway_bin_test", + ":openshell-gateway_lib_test", + ], + visibility = ["//crates:__pkg__"], +) diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml new file mode 100644 index 0000000000..3f393c850f --- /dev/null +++ b/crates/openshell-gateway/Cargo.toml @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-gateway" +description = "OpenShell gateway binary composition" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "openshell-gateway" +path = "src/main.rs" + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false } +openshell-server = { path = "../openshell-server", default-features = false } +openshell-otel = { path = "../openshell-otel", optional = true } +async-trait = "0.1" +miette = { workspace = true } +tokio = { workspace = true } + +[target.'cfg(not(target_os = "windows"))'.dependencies] +openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } +openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes", optional = true } +openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } +hyper-util = { workspace = true, optional = true } +nix = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +rustix = { workspace = true, optional = true } +tonic = { workspace = true, optional = true } +tower = { workspace = true, optional = true } +tracing = { workspace = true, optional = true } + +[features] +default = ["telemetry", "in-tree-compute-drivers"] +in-tree-compute-drivers = [ + "dep:openshell-driver-docker", + "dep:openshell-driver-kubernetes", + "dep:openshell-driver-podman", + "dep:openshell-otel", + "dep:hyper-util", + "dep:nix", + "dep:serde", + "dep:rustix", + "dep:tonic", + "dep:tower", + "dep:tracing", +] +telemetry = ["openshell-core/telemetry", "openshell-server/telemetry"] +## Convenience alias: every default feature except `telemetry`. Build a +## telemetry-free gateway with +## `--no-default-features --features defaults-without-telemetry` and stay +## correct as new default features are added. Cargo cannot subtract a single +## default feature, so this alias must be paired with `--no-default-features`; +## enabling it alongside `telemetry` is a compile error rather than a silent +## telemetry-on build. Kept in sync with `default` by +## `rust:verify:defaults-without-telemetry`. +defaults-without-telemetry = ["in-tree-compute-drivers"] +bundled-z3 = ["openshell-server/bundled-z3"] + +[lints] +workspace = true + +[dev-dependencies] +tempfile = "3" diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs new file mode 100644 index 0000000000..c292c208a4 --- /dev/null +++ b/crates/openshell-gateway/src/lib.rs @@ -0,0 +1,299 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Standard gateway binary composition. +//! +//! The server remains backend-agnostic. This crate is the composition boundary +//! that links first-party compute drivers into the distributed gateway binary. + +// `defaults-without-telemetry` is an alias for the default feature set minus +// `telemetry`, not a switch that turns telemetry off. Cargo cannot subtract a +// default feature, so adding it on top of the defaults would otherwise produce +// a telemetry-on build that reads as telemetry-free. Fail the build instead. +#[cfg(all(feature = "telemetry", feature = "defaults-without-telemetry"))] +compile_error!( + "features `telemetry` and `defaults-without-telemetry` are mutually exclusive; \ + build a telemetry-free gateway with `--no-default-features --features defaults-without-telemetry`" +); + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +mod vm; + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +use openshell_server::ComputeDriverRegistration; +use openshell_server::ComputeDriverRegistry; + +/// Install every first-party compute driver linked into the standard gateway. +#[must_use] +pub fn install_default_compute_drivers() -> ComputeDriverRegistry { + #[allow(unused_mut)] + let mut registry = ComputeDriverRegistry::new(); + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] + install_in_tree_compute_drivers(&mut registry); + registry +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { + for registration in [ + ComputeDriverRegistration::new( + "kubernetes", + 100, + Some(|| std::env::var_os("KUBERNETES_SERVICE_HOST").is_some()), + KubernetesFactory, + ) + .map(|registration| { + registration + .without_mtls_user_auth() + .with_inherited_config_keys(&[ + "namespace", + "default_image", + "supervisor_image", + "client_tls_secret_name", + "service_account_name", + "host_gateway_ip", + "enable_user_namespaces", + "sa_token_ttl_secs", + ]) + }), + ComputeDriverRegistration::new( + "podman", + 200, + Some(openshell_driver_podman::driver::is_available), + PodmanFactory, + ) + .map(|registration| { + registration + .with_local_singleplayer() + .with_tracing_setup(podman_tracing_setup) + .with_inherited_config_keys(&[ + "default_image", + "supervisor_image", + "host_gateway_ip", + "guest_tls_ca", + "guest_tls_cert", + "guest_tls_key", + ]) + }), + ComputeDriverRegistration::new( + "docker", + 300, + Some(openshell_driver_docker::is_available), + DockerFactory, + ) + .map(|registration| { + registration + .with_local_singleplayer() + .with_inherited_config_keys(&[ + "sandbox_namespace", + "default_image", + "supervisor_image", + "host_gateway_ip", + "guest_tls_ca", + "guest_tls_cert", + "guest_tls_key", + ]) + }), + ComputeDriverRegistration::new("vm", u16::MAX, None, VmFactory).map(|registration| { + registration + .with_local_singleplayer() + .with_inherited_config_keys(&[ + "default_image", + "guest_tls_ca", + "guest_tls_cert", + "guest_tls_key", + ]) + }), + ] { + registry + .install(registration.expect("first-party driver name is valid")) + .expect("first-party driver names are unique"); + } +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn podman_tracing_setup( + otlp_endpoint: Option<&str>, +) -> openshell_server::ComputeDriverTracingSetup { + let (provider, error) = openshell_driver_podman::otel_tracing::provider_for(otlp_endpoint); + let layer = provider.as_ref().map(|provider| { + let layer: openshell_server::ComputeDriverTracingLayer = Box::new( + openshell_driver_podman::otel_tracing::in_process_layer( + provider, + ), + ); + layer + }); + let shutdown = provider.map(|provider| { + let shutdown: openshell_server::ComputeDriverTracingShutdown = Box::new(move || { + provider + .shutdown() + .map_err(|error| error.to_string()) + }); + shutdown + }); + openshell_server::ComputeDriverTracingSetup::new( + layer, + shutdown, + error.map(|error| error.to_string()), + Some(openshell_driver_podman::otel_tracing::IN_PROCESS_TARGET_PREFIX), + ) +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[derive(Clone, Copy)] +struct KubernetesFactory; + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[async_trait::async_trait] +impl openshell_server::ComputeDriverFactory for KubernetesFactory { + async fn build( + &self, + context: openshell_server::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + let mut config: openshell_driver_kubernetes::KubernetesComputeConfig = + context.driver_config()?; + if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { + config.workspace_default_storage_size = size; + } + if let Ok(storage_class) = std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") { + config.workspace_storage_class = storage_class; + } + let driver = openshell_driver_kubernetes::KubernetesComputeDriver::new( + config, + context.shutdown_receiver(), + ) + .await + .map_err(|error| openshell_core::Error::execution(error.to_string()))?; + let driver = openshell_driver_kubernetes::ComputeDriverService::new(driver); + Ok(openshell_server::ComputeDriverInstance::InProcess( + std::sync::Arc::new(driver), + )) + } +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[derive(Clone, Copy)] +struct DockerFactory; + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[async_trait::async_trait] +impl openshell_server::ComputeDriverFactory for DockerFactory { + async fn build( + &self, + context: openshell_server::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + let mut config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?; + apply_guest_tls( + &mut config.guest_tls_ca, + &mut config.guest_tls_cert, + &mut config.guest_tls_key, + context.guest_tls_paths(), + ); + let driver = openshell_driver_docker::DockerComputeDriver::new( + context.gateway_bind_address(), + context.gateway_log_level(), + &config, + ) + .await + .map_err(|error| openshell_core::Error::execution(error.to_string()))?; + Ok(openshell_server::ComputeDriverInstance::InProcess( + std::sync::Arc::new(driver), + )) + } +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[derive(Clone, Copy)] +struct PodmanFactory; + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[async_trait::async_trait] +impl openshell_server::ComputeDriverFactory for PodmanFactory { + async fn build( + &self, + context: openshell_server::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + let mut config: openshell_driver_podman::PodmanComputeConfig = context.driver_config()?; + config.gateway_port = context.gateway_port(); + if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") { + config.socket_path = Some(path.into()); + } + if let Ok(ip) = std::env::var("OPENSHELL_PODMAN_HOST_GATEWAY_IP") { + config.host_gateway_ip = ip; + } + if let Ok(mode) = std::env::var("OPENSHELL_PODMAN_USERNS") { + config.userns = Some(mode); + } + apply_guest_tls( + &mut config.guest_tls_ca, + &mut config.guest_tls_cert, + &mut config.guest_tls_key, + context.guest_tls_paths(), + ); + let driver = openshell_driver_podman::PodmanComputeDriver::new(config) + .await + .map_err(|error| openshell_core::Error::execution(error.to_string()))?; + let driver = openshell_driver_podman::ComputeDriverService::new(driver); + Ok(openshell_server::ComputeDriverInstance::InProcess( + std::sync::Arc::new(driver), + )) + } +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[derive(Clone, Copy)] +struct VmFactory; + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[async_trait::async_trait] +impl openshell_server::ComputeDriverFactory for VmFactory { + async fn build( + &self, + context: openshell_server::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + let mut config: vm::VmComputeConfig = context.driver_config()?; + if config.state_dir.as_os_str().is_empty() { + config.state_dir = vm::VmComputeConfig::default_state_dir(); + } + if config.grpc_endpoint.trim().is_empty() + && (!context.gateway_tls_enabled() || context.guest_tls_paths().is_some()) + { + let scheme = if context.gateway_tls_enabled() { + "https" + } else { + "http" + }; + config.grpc_endpoint = format!("{scheme}://127.0.0.1:{}", context.gateway_port()); + } + apply_guest_tls( + &mut config.guest_tls_ca, + &mut config.guest_tls_cert, + &mut config.guest_tls_key, + context.guest_tls_paths(), + ); + let endpoint = + vm::spawn(context.gateway_log_level(), &config, context.otlp_config()).await?; + Ok(openshell_server::ComputeDriverInstance::ManagedRemote( + endpoint, + )) + } +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn apply_guest_tls( + ca: &mut Option, + cert: &mut Option, + key: &mut Option, + defaults: Option<(&std::path::Path, &std::path::Path, &std::path::Path)>, +) { + if ca.is_none() + && cert.is_none() + && key.is_none() + && let Some((default_ca, default_cert, default_key)) = defaults + { + *ca = Some(default_ca.to_owned()); + *cert = Some(default_cert.to_owned()); + *key = Some(default_key.to_owned()); + } +} diff --git a/crates/openshell-server/src/main.rs b/crates/openshell-gateway/src/main.rs similarity index 59% rename from crates/openshell-server/src/main.rs rename to crates/openshell-gateway/src/main.rs index c76761016d..85d0611867 100644 --- a/crates/openshell-server/src/main.rs +++ b/crates/openshell-gateway/src/main.rs @@ -1,14 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! `OpenShell` Gateway binary entrypoint. - -use miette::Result; - #[tokio::main] -async fn main() -> Result<()> { +async fn main() -> miette::Result<()> { openshell_server::cli::run_cli_with_compute_drivers( - openshell_server::install_default_compute_drivers(), + openshell_gateway::install_default_compute_drivers(), ) .await } diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-gateway/src/vm.rs similarity index 94% rename from crates/openshell-server/src/compute/vm.rs rename to crates/openshell-gateway/src/vm.rs index 6a66fc8aa5..69fbdebcbf 100644 --- a/crates/openshell-server/src/compute/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -29,19 +29,19 @@ //! trait implementation registering the VM driver against the generic //! interface. -use super::AcquiredRemoteDriverEndpoint; -#[cfg(unix)] -use super::ManagedDriverProcess; -use crate::config_file::OtlpConfig; -#[cfg(unix)] -use crate::otel_tracing::TraceContextInterceptor; #[cfg(unix)] use hyper_util::rt::TokioIo; #[cfg(unix)] use openshell_core::proto::compute::v1::{ GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, }; -use openshell_core::{ComputeDriverKind, Config, Error, Result}; +use openshell_core::{Error, Result}; +#[cfg(unix)] +use openshell_otel::TraceContextInterceptor; +use openshell_server::AcquiredRemoteDriverEndpoint; +#[cfg(unix)] +use openshell_server::ManagedDriverProcess; +use openshell_server::config_file::OtlpConfig; #[cfg(unix)] use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; #[cfg(unix)] @@ -453,7 +453,8 @@ pub fn compute_driver_guest_tls_paths( /// kills the subprocess and removes the socket on drop. #[cfg(unix)] pub async fn spawn( - config: &Config, + gateway_log_level: &str, + gateway_name: &str, vm_config: &VmComputeConfig, otlp_config: Option<&OtlpConfig>, ) -> Result { @@ -477,8 +478,8 @@ pub async fn spawn( command .arg("--expected-peer-pid") .arg(std::process::id().to_string()); - command.arg("--log-level").arg(&config.log_level); - append_otlp_args(&mut command, otlp_config, &config.name); + command.arg("--log-level").arg(gateway_log_level); + append_otlp_args(&mut command, otlp_config, gateway_name); command .arg("--openshell-endpoint") .arg(&vm_config.grpc_endpoint); @@ -513,10 +514,8 @@ pub async fn spawn( })?; let channel = wait_for_compute_driver(&socket_path, &mut child).await?; let process = Arc::new(ManagedDriverProcess::new(child, socket_path)); - Ok(AcquiredRemoteDriverEndpoint::managed_builtin( - ComputeDriverKind::Vm, - channel, - process, + Ok(AcquiredRemoteDriverEndpoint::managed( + "vm", channel, process, )) } @@ -530,7 +529,8 @@ fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>, gat #[cfg(not(unix))] pub async fn spawn( - _config: &Config, + _gateway_log_level: &str, + _gateway_name: &str, _vm_config: &VmComputeConfig, _otlp_config: Option<&OtlpConfig>, ) -> Result { @@ -613,9 +613,8 @@ mod tests { VmComputeConfig, append_otlp_args, compute_driver_guest_tls_paths, compute_driver_socket_path, current_euid, prepare_compute_driver_socket_path, prepare_vm_state_dir, resolve_compute_driver_bin, resolve_driver_search_dirs, - wait_for_compute_driver, }; - use crate::config_file::OtlpConfig; + use openshell_server::config_file::OtlpConfig; use std::os::unix::fs::PermissionsExt; use std::os::unix::net::UnixListener as StdUnixListener; use std::path::PathBuf; @@ -649,43 +648,6 @@ mod tests { ); } - #[tokio::test] - async fn readiness_probe_propagates_the_active_trace() { - use crate::otel_tracing::test_exporter; - use crate::test_support::FakeComputeDriver; - - let dir = tempdir().unwrap(); - let socket_path = dir.path().join("compute-driver.sock"); - let driver = FakeComputeDriver::new(); - let _server = driver.serve_uds(&socket_path).unwrap(); - let mut child = tokio::process::Command::new("sh") - .arg("-c") - .arg("read _") - .stdin(std::process::Stdio::piped()) - .kill_on_drop(true) - .spawn() - .unwrap(); - - let traced = test_exporter::install_traced(); - wait_for_compute_driver(&socket_path, &mut child) - .await - .unwrap(); - - let readiness = traced.spans_named("driver.wait_for_ready"); - assert_eq!(readiness.len(), 1, "one readiness operation should finish"); - test_exporter::assert_is_root(&readiness[0]); - let trace_id = readiness[0].span_context.trace_id().to_string(); - assert_eq!( - driver.traceparents().len(), - 1, - "the readiness capability probe should carry trace context" - ); - assert!( - driver.traceparents()[0].contains(&trace_id), - "the readiness probe should be part of the active trace" - ); - } - #[test] fn resolve_driver_bin_uses_driver_dir_when_binary_present() { let dir = tempdir().unwrap(); diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 1cfd214f20..0f2bc8abb0 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -10,10 +10,6 @@ rust-version.workspace = true license.workspace = true repository.workspace = true -[[bin]] -name = "openshell-gateway" -path = "src/main.rs" - [dependencies] openshell-bootstrap = { path = "../openshell-bootstrap" } openshell-core = { path = "../openshell-core", default-features = false, features = ["oauth"] } @@ -115,38 +111,12 @@ x509-parser = "0.16" arc-swap = "1" notify = "8" -[target.'cfg(not(target_os = "windows"))'.dependencies] -openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } -openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes", optional = true } -openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } - -# MXC is the Windows-only in-process compute backend (openshell-driver-mxc is a -# no-op stub on other targets). It is only linked into the gateway on Windows. -[target.'cfg(target_os = "windows")'.dependencies] -openshell-driver-mxc = { path = "../openshell-driver-mxc" } - [features] -default = ["telemetry", "in-tree-compute-drivers"] -## Link the first-party compute drivers into the standard gateway binary. -## Disable this feature for a protocol-only gateway that uses external drivers. -in-tree-compute-drivers = [ - "dep:openshell-driver-docker", - "dep:openshell-driver-kubernetes", - "dep:openshell-driver-podman", -] +default = ["telemetry"] ## Compile in anonymous telemetry emission (forwards to openshell-core/telemetry). ## On by default; build with `--no-default-features` for a telemetry-free gateway ## that contains no telemetry endpoint, HTTP client, or emission code. telemetry = ["openshell-core/telemetry"] -## Convenience alias: every default feature except `telemetry`. Build a -## telemetry-free gateway with -## `--no-default-features --features defaults-without-telemetry` and stay -## correct as new default features are added. Cargo cannot subtract a single -## default feature, so this alias must be paired with `--no-default-features`; -## enabling it alongside `telemetry` is a compile error rather than a silent -## telemetry-on build. Kept in sync with `default` by -## `rust:verify:defaults-without-telemetry`. -defaults-without-telemetry = ["in-tree-compute-drivers"] bundled-z3 = ["openshell-prover/bundled-z3"] test-support = [] diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index f22e1355e4..f18dbe07aa 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -6,7 +6,6 @@ use clap::parser::ValueSource; use clap::{ArgAction, ArgMatches, Command, CommandFactory, FromArgMatches, Parser}; use miette::{IntoDiagnostic, Result}; -use openshell_core::ComputeDriverKind; use openshell_core::config::{DEFAULT_GATEWAY_NAME, DEFAULT_SERVER_PORT}; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; @@ -17,10 +16,7 @@ use crate::certgen; use crate::compute::driver_config::GuestTlsPaths; use crate::config_file::{self, ConfigFile, GatewayFileSection}; use crate::defaults::{self, LocalTlsPaths}; -use crate::{ - ComputeDriverRegistry, ServerStartupConfig, configured_compute_driver_for_startup, - install_default_compute_drivers, run_server, tracing_bus::TracingLogBus, -}; +use crate::{ComputeDriverRegistry, ServerStartupConfig, run_server, tracing_bus::TracingLogBus}; /// `OpenShell` gateway process - gRPC and HTTP server with protocol multiplexing. /// @@ -107,12 +103,10 @@ struct RunArgs { /// Compute drivers configured for this gateway. /// - /// Accepts a comma-delimited list such as `kubernetes` or - /// `kubernetes,podman`. The configuration format is future-proofed for - /// multiple drivers, but the gateway currently requires exactly one. - /// When unset, the gateway auto-detects the driver based on the runtime - /// environment (Kubernetes → Podman → Docker). VM is never - /// auto-detected and requires explicit configuration. + /// Accepts a comma-delimited list of registered driver names. The + /// configuration format is future-proofed for multiple drivers, but the + /// gateway currently requires exactly one. When unset, the gateway runs + /// detection probes supplied by the drivers compiled into the binary. #[arg( long, alias = "driver", @@ -127,9 +121,9 @@ struct RunArgs { /// /// When set, the socket is associated with the single driver name supplied /// by `--drivers` or `OPENSHELL_DRIVERS` and replaces normal construction - /// for that selected name, including canonical built-in names. The gateway - /// connects to this operator-provided endpoint; it does not provision the - /// remote driver. + /// for that selected name, including a compiled registration with the same + /// name. The gateway connects to this operator-provided endpoint; it does + /// not provision the remote driver. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] compute_driver_socket: Option, @@ -147,9 +141,9 @@ struct RunArgs { /// Enable mTLS client certificate authentication for local single-user gateways. /// - /// When unset, this defaults on for Docker, Podman, and VM gateways that - /// have client certificate verification configured and no OIDC issuer. - /// Kubernetes deployments must use OIDC or fronting-proxy auth instead. + /// When unset, this defaults on for drivers registered as local + /// single-player backends when client certificate verification is + /// configured and no OIDC issuer is present. #[arg( long = "enable-mtls-auth", env = "OPENSHELL_ENABLE_MTLS_AUTH", @@ -231,7 +225,7 @@ pub fn command() -> Command { } pub async fn run_cli() -> Result<()> { - run_cli_with_compute_drivers(install_default_compute_drivers()).await + run_cli_with_compute_drivers(ComputeDriverRegistry::new()).await } /// Run the gateway CLI with the compute drivers linked by the binary. @@ -249,7 +243,12 @@ pub async fn run_cli_with_compute_drivers(compute_drivers: ComputeDriverRegistry } } -fn prepare_server_config( +#[cfg(test)] +fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result { + prepare_server_config_with_drivers(args, matches, &ComputeDriverRegistry::new()) +} + +fn prepare_server_config_with_drivers( args: &mut RunArgs, matches: &ArgMatches, compute_drivers: &ComputeDriverRegistry, @@ -270,7 +269,7 @@ fn prepare_server_config( let compute_driver = compute_drivers .select(&args.drivers) .map_err(|error| miette::miette!("{error}"))?; - let compute_driver_kind = compute_driver.name().parse::().ok(); + let selected_registration = compute_drivers.get(compute_driver.name()); let local_tls = apply_runtime_defaults(args)?; let guest_tls = local_tls.as_ref().map(GuestTlsPaths::from); @@ -281,7 +280,7 @@ fn prepare_server_config( let has_client_ca = args.tls_client_ca.is_some(); let has_oidc = args.oidc_issuer.is_some(); let mtls_auth_enabled = - resolve_mtls_auth_enabled(args, matches, file.as_ref(), compute_driver_kind); + resolve_mtls_auth_enabled(args, matches, file.as_ref(), selected_registration); if args.disable_tls && has_client_ca { return Err(miette::miette!( @@ -298,9 +297,11 @@ fn prepare_server_config( "mTLS user authentication requires --tls-client-ca so client certificates can be verified." )); } - if mtls_auth_enabled && matches!(compute_driver_kind, Some(ComputeDriverKind::Kubernetes)) { + if mtls_auth_enabled + && selected_registration.is_some_and(|registration| !registration.supports_mtls_user_auth()) + { return Err(miette::miette!( - "mTLS user authentication is not supported with the Kubernetes compute driver. Configure OIDC or a trusted fronting proxy for user authentication." + "mTLS user authentication is not supported with the selected compute driver. Configure OIDC or a trusted fronting proxy for user authentication." )); } @@ -501,8 +502,7 @@ async fn run_from_args( matches: ArgMatches, compute_drivers: ComputeDriverRegistry, ) -> Result<()> { - let prepared = prepare_server_config(&mut args, &matches, &compute_drivers)?; - let compute_driver = configured_compute_driver_for_startup(&compute_drivers, &prepared)?; + let prepared = prepare_server_config_with_drivers(&mut args, &matches, &compute_drivers)?; let tracing_log_bus = TracingLogBus::new(); let otlp_config = prepared @@ -511,14 +511,20 @@ async fn run_from_args( .and_then(|f| f.openshell.gateway.otlp.as_ref()); let gateway_resource = crate::otel_tracing::GatewayResourceAttributes::new( Some(prepared.config.name.as_str()), - Some(compute_driver.name()), + Some(prepared.compute_driver.name()), + ); + let compute_driver_tracing = compute_drivers.tracing_setup( + &prepared.compute_driver, + &prepared.config.compute_driver_endpoints, + otlp_config.map(|config| config.endpoint.as_str()), + Some(prepared.config.name.as_str()), ); let (tracing_handle, setup_error) = crate::tracing_setup::install( EnvFilter::try_from_default_env() .unwrap_or_else(|_| EnvFilter::new(&prepared.config.log_level)), &tracing_log_bus, otlp_config, - &compute_driver, + compute_driver_tracing, gateway_resource, ); @@ -576,7 +582,7 @@ async fn run_from_args( info!(bind = %prepared.config.bind_address, "Starting OpenShell server"); - let result = Box::pin(run_server(prepared, compute_driver, tracing_log_bus)).await; + let result = Box::pin(run_server(prepared, tracing_log_bus, compute_drivers)).await; tracing_handle.shutdown(); @@ -817,23 +823,15 @@ fn normalize_compute_driver_socket_args(args: &mut RunArgs, matches: &ArgMatches } } -fn is_singleplayer_driver(driver: Option) -> bool { - matches!( - driver, - Some( - ComputeDriverKind::Docker - | ComputeDriverKind::Podman - | ComputeDriverKind::Vm - | ComputeDriverKind::Mxc - ) - ) +fn is_singleplayer_driver(registration: Option<&crate::ComputeDriverRegistration>) -> bool { + registration.is_some_and(crate::ComputeDriverRegistration::is_local_singleplayer) } fn resolve_mtls_auth_enabled( args: &RunArgs, matches: &ArgMatches, file: Option<&ConfigFile>, - compute_driver: Option, + selected_registration: Option<&crate::ComputeDriverRegistration>, ) -> bool { let file_configured = file .and_then(|f| f.openshell.gateway.mtls_auth.as_ref()) @@ -846,7 +844,7 @@ fn resolve_mtls_auth_enabled( return false; } - is_singleplayer_driver(compute_driver) + is_singleplayer_driver(selected_registration) } #[cfg(test)] @@ -859,37 +857,49 @@ mod tests { static REGISTRY_DETECTION_CALLS: AtomicUsize = AtomicUsize::new(0); - fn detect_registered_docker() -> bool { + fn detect_registered_local() -> bool { REGISTRY_DETECTION_CALLS.fetch_add(1, Ordering::SeqCst); true } #[derive(Clone, Copy)] - struct TestComputeDriverFactory; + struct TestFactory; #[async_trait::async_trait] - impl crate::ComputeDriverFactory for TestComputeDriverFactory { + impl crate::ComputeDriverFactory for TestFactory { async fn build( &self, _context: crate::ComputeDriverBuildContext<'_>, - ) -> openshell_core::Result { - unreachable!("configuration tests do not construct the driver") + ) -> openshell_core::Result { + unreachable!("CLI metadata tests do not build drivers") } } - fn detected_docker_registry() -> crate::ComputeDriverRegistry { + fn test_registry(name: &str, singleplayer: bool, mtls: bool) -> crate::ComputeDriverRegistry { + let mut registration = + crate::ComputeDriverRegistration::new(name, 100, None, TestFactory).unwrap(); + if singleplayer { + registration = registration.with_local_singleplayer(); + } + if !mtls { + registration = registration.without_mtls_user_auth(); + } let mut registry = crate::ComputeDriverRegistry::new(); + registry.install(registration).unwrap(); registry - .install( - crate::ComputeDriverRegistration::new( - "docker", - 100, - Some(detect_registered_docker), - TestComputeDriverFactory, - ) - .unwrap(), - ) - .unwrap(); + } + + fn detected_local_registry() -> crate::ComputeDriverRegistry { + let registration = crate::ComputeDriverRegistration::new( + "local", + 100, + Some(detect_registered_local), + TestFactory, + ) + .unwrap() + .with_local_singleplayer(); + let mut registry = crate::ComputeDriverRegistry::new(); + registry.install(registration).unwrap(); registry } @@ -1350,7 +1360,7 @@ mod tests { "--db-url", "sqlite::memory:", "--drivers", - "docker", + "local", "--tls-cert", "/tmp/server.crt", "--tls-key", @@ -1363,7 +1373,7 @@ mod tests { &args, &matches, None, - Some(openshell_core::ComputeDriverKind::Docker) + test_registry("local", true, true).get("local") )); } @@ -1376,7 +1386,6 @@ mod tests { let config = tempfile::tempdir().unwrap(); let _state = EnvVarGuard::set("XDG_STATE_HOME", state.path().to_str().unwrap()); let _config = EnvVarGuard::set("XDG_CONFIG_HOME", config.path().to_str().unwrap()); - let _kubernetes = EnvVarGuard::set("KUBERNETES_SERVICE_HOST", "10.0.0.1"); let _mtls = EnvVarGuard::remove("OPENSHELL_ENABLE_MTLS_AUTH"); let _drivers = EnvVarGuard::remove("OPENSHELL_DRIVERS"); REGISTRY_DETECTION_CALLS.store(0, Ordering::SeqCst); @@ -1392,18 +1401,19 @@ mod tests { "--tls-client-ca", "/tmp/ca.crt", ]); - let registry = detected_docker_registry(); + let registry = detected_local_registry(); - let prepared = super::prepare_server_config(&mut args, &matches, ®istry).unwrap(); + let prepared = + super::prepare_server_config_with_drivers(&mut args, &matches, ®istry).unwrap(); - assert_eq!(prepared.compute_driver.name(), "docker"); + assert_eq!(prepared.compute_driver.name(), "local"); assert!(prepared.config.compute_drivers.is_empty()); assert!(prepared.config.mtls_auth.enabled); assert_eq!(REGISTRY_DETECTION_CALLS.load(Ordering::SeqCst), 1); } #[test] - fn mtls_auth_does_not_auto_default_for_kubernetes_driver() { + fn mtls_auth_does_not_auto_default_for_shared_driver() { let _lock = ENV_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -1414,7 +1424,7 @@ mod tests { "--db-url", "sqlite::memory:", "--drivers", - "kubernetes", + "shared", "--tls-cert", "/tmp/server.crt", "--tls-key", @@ -1427,7 +1437,7 @@ mod tests { &args, &matches, None, - Some(openshell_core::ComputeDriverKind::Kubernetes) + test_registry("shared", false, false).get("shared") )); } @@ -1443,7 +1453,7 @@ mod tests { "--db-url", "sqlite::memory:", "--drivers", - "docker", + "local", "--tls-cert", "/tmp/server.crt", "--tls-key", @@ -1464,7 +1474,7 @@ enabled = false &args, &matches, Some(&file), - Some(openshell_core::ComputeDriverKind::Docker) + test_registry("local", true, true).get("local") )); } @@ -1702,23 +1712,12 @@ ssh_session_ttl_secs = 1234 } #[test] - fn singleplayer_driver_matches_only_one_local_driver() { - for driver in [ - openshell_core::ComputeDriverKind::Docker, - openshell_core::ComputeDriverKind::Podman, - openshell_core::ComputeDriverKind::Vm, - openshell_core::ComputeDriverKind::Mxc, - ] { - assert!( - super::is_singleplayer_driver(Some(driver)), - "{driver} should be singleplayer" - ); - } + fn singleplayer_behavior_comes_from_registration() { + let local = test_registry("local", true, true); + assert!(super::is_singleplayer_driver(local.get("local"))); - assert!(!super::is_singleplayer_driver(Some( - openshell_core::ComputeDriverKind::Kubernetes - ))); - assert!(!super::is_singleplayer_driver(None)); + let shared = test_registry("shared", false, true); + assert!(!super::is_singleplayer_driver(shared.get("shared"))); } #[test] @@ -1744,11 +1743,6 @@ ssh_session_ttl_secs = 1234 Some(std::path::Path::new("/run/openshell/kyma.sock")) ); assert_eq!(args.drivers, ["kyma"]); - assert!( - args.drivers[0] - .parse::() - .is_err() - ); } #[test] @@ -1928,12 +1922,8 @@ mem_mib = "not-a-number" "--disable-tls", ]); - let prepared = super::prepare_server_config( - &mut args, - &matches, - &crate::install_default_compute_drivers(), - ) - .expect("server config is prepared"); + let prepared = + super::prepare_server_config(&mut args, &matches).expect("server config is prepared"); assert_eq!(prepared.config.compute_drivers, vec!["podman".to_string()]); assert_eq!( @@ -1944,53 +1934,4 @@ mem_mib = "not-a-number" assert!(file.openshell.drivers.contains_key("docker")); assert!(file.openshell.drivers.contains_key("vm")); } - - #[test] - #[cfg(not(target_os = "windows"))] - fn driver_inherits_shared_image_from_gateway_section() { - // [openshell.gateway].default_image inherits into the K8s driver - // table when the driver-specific table does not set it. - let file = config_file_from_toml( - r#" -[openshell.gateway] -default_image = "ghcr.io/nvidia/openshell/sandbox:1.0" - -[openshell.drivers.kubernetes] -namespace = "agents" -"#, - ); - let merged = crate::config_file::driver_table( - super::ComputeDriverKind::Kubernetes.as_str(), - &file.openshell.gateway, - file.openshell.drivers.get("kubernetes"), - ); - let parsed = merged - .try_into::() - .expect("merged table deserializes"); - assert_eq!(parsed.default_image, "ghcr.io/nvidia/openshell/sandbox:1.0"); - assert_eq!(parsed.namespace, "agents"); - } - - #[test] - #[cfg(not(target_os = "windows"))] - fn driver_specific_value_overrides_gateway_inheritance() { - let file = config_file_from_toml( - r#" -[openshell.gateway] -default_image = "gateway-default:1.0" - -[openshell.drivers.kubernetes] -default_image = "k8s-specific:1.0" -"#, - ); - let merged = crate::config_file::driver_table( - super::ComputeDriverKind::Kubernetes.as_str(), - &file.openshell.gateway, - file.openshell.drivers.get("kubernetes"), - ); - let parsed = merged - .try_into::() - .expect("deserializes"); - assert_eq!(parsed.default_image, "k8s-specific:1.0"); - } } diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index c25c63ded2..779fcd4eed 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -7,9 +7,6 @@ //! driver-specific environment overrides, and applying gateway startup defaults. //! It does not acquire, connect to, or start compute drivers. -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub mod builtin; - use crate::config_file; use crate::defaults::LocalTlsPaths; #[cfg(target_os = "windows")] @@ -91,16 +88,18 @@ pub struct RemoteDriverConfig { pub fn driver_config_from_context( context: DriverStartupContext<'_>, driver_name: &str, + inherited_config_keys: &[&str], ) -> Result where T: Default + serde::de::DeserializeOwned, { - driver_config_from_file(context.file, driver_name) + driver_config_from_file(context.file, driver_name, inherited_config_keys) } fn driver_config_from_file( file: Option<&config_file::ConfigFile>, driver_name: &str, + inherited_config_keys: &[&str], ) -> Result where T: Default + serde::de::DeserializeOwned, @@ -108,10 +107,11 @@ where let Some(file) = file else { return Ok(T::default()); }; - let merged = config_file::driver_table( + let merged = config_file::driver_table_with_inherited_keys( driver_name, &file.openshell.gateway, file.openshell.drivers.get(driver_name), + inherited_config_keys, ); merged.try_into().map_err(|e| { Error::config(format!( diff --git a/crates/openshell-server/src/compute/driver_config/builtin.rs b/crates/openshell-server/src/compute/driver_config/builtin.rs deleted file mode 100644 index dea867237d..0000000000 --- a/crates/openshell-server/src/compute/driver_config/builtin.rs +++ /dev/null @@ -1,231 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Configuration construction for built-in compute drivers. - -use super::{DriverStartupContext, GuestTlsPaths, driver_config_from_context}; -use crate::compute::VmComputeConfig; -#[cfg(test)] -use crate::config_file; -use openshell_core::{ComputeDriverKind, Result}; -use openshell_driver_docker::DockerComputeConfig; -use openshell_driver_kubernetes::KubernetesComputeConfig; -use openshell_driver_podman::PodmanComputeConfig; -use std::path::PathBuf; - -/// Build the selected Kubernetes config from TOML plus runtime defaults. -pub fn kubernetes_config_from_context( - context: DriverStartupContext<'_>, -) -> Result { - let mut cfg = driver_config_from_context(context, ComputeDriverKind::Kubernetes.as_str())?; - apply_kubernetes_runtime_defaults(&mut cfg); - Ok(cfg) -} - -/// Build the selected Podman config from TOML plus runtime defaults. -pub fn podman_config_from_context( - context: DriverStartupContext<'_>, -) -> Result { - let mut podman = driver_config_from_context(context, ComputeDriverKind::Podman.as_str())?; - apply_podman_runtime_defaults(&mut podman, context); - Ok(podman) -} - -/// Build the selected Docker config from TOML plus runtime defaults. -pub fn docker_config_from_context( - context: DriverStartupContext<'_>, -) -> Result { - let mut cfg = driver_config_from_context(context, ComputeDriverKind::Docker.as_str())?; - apply_docker_runtime_defaults(&mut cfg, context); - Ok(cfg) -} - -/// Build the selected VM config from TOML plus runtime defaults. -pub fn vm_config_from_context(context: DriverStartupContext<'_>) -> Result { - let mut cfg = driver_config_from_context(context, ComputeDriverKind::Vm.as_str())?; - apply_vm_runtime_defaults(&mut cfg, context); - Ok(cfg) -} - -fn apply_kubernetes_runtime_defaults(k8s: &mut KubernetesComputeConfig) { - if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { - k8s.workspace_default_storage_size = size; - } - if let Ok(storage_class) = std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") { - k8s.workspace_storage_class = storage_class; - } -} - -fn apply_podman_runtime_defaults( - podman: &mut PodmanComputeConfig, - context: DriverStartupContext<'_>, -) { - podman.gateway_port = context.gateway_port; - apply_podman_env_overrides(podman); - apply_guest_tls_defaults_to_split_fields( - &mut podman.guest_tls_ca, - &mut podman.guest_tls_cert, - &mut podman.guest_tls_key, - context.guest_tls, - ); -} - -fn apply_docker_runtime_defaults(cfg: &mut DockerComputeConfig, context: DriverStartupContext<'_>) { - apply_guest_tls_defaults_to_split_fields( - &mut cfg.guest_tls_ca, - &mut cfg.guest_tls_cert, - &mut cfg.guest_tls_key, - context.guest_tls, - ); -} - -fn apply_vm_runtime_defaults(cfg: &mut VmComputeConfig, context: DriverStartupContext<'_>) { - if cfg.state_dir.as_os_str().is_empty() { - cfg.state_dir = VmComputeConfig::default_state_dir(); - } - if cfg.grpc_endpoint.trim().is_empty() - && (!context.gateway_tls_enabled || context.guest_tls.is_some()) - { - let scheme = if context.gateway_tls_enabled { - "https" - } else { - "http" - }; - cfg.grpc_endpoint = format!("{scheme}://127.0.0.1:{}", context.gateway_port); - } - - apply_guest_tls_defaults_to_split_fields( - &mut cfg.guest_tls_ca, - &mut cfg.guest_tls_cert, - &mut cfg.guest_tls_key, - context.guest_tls, - ); -} - -fn apply_guest_tls_defaults_to_split_fields( - ca: &mut Option, - cert: &mut Option, - key: &mut Option, - defaults: Option<&GuestTlsPaths>, -) { - if ca.is_none() - && cert.is_none() - && key.is_none() - && let Some(paths) = defaults - { - *ca = Some(paths.ca.clone()); - *cert = Some(paths.cert.clone()); - *key = Some(paths.key.clone()); - } -} - -fn apply_podman_env_overrides(podman: &mut PodmanComputeConfig) { - if let Ok(p) = std::env::var("OPENSHELL_PODMAN_SOCKET") { - podman.socket_path = Some(PathBuf::from(p)); - } - if let Ok(ip) = std::env::var("OPENSHELL_PODMAN_HOST_GATEWAY_IP") { - podman.host_gateway_ip = ip; - } - if let Ok(mode) = std::env::var("OPENSHELL_PODMAN_USERNS") { - podman.userns = Some(mode); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::BTreeMap; - - fn test_context(file: Option<&config_file::ConfigFile>) -> DriverStartupContext<'_> { - static EMPTY_ENDPOINT_OVERRIDES: std::sync::LazyLock> = - std::sync::LazyLock::new(BTreeMap::new); - DriverStartupContext { - file, - guest_tls: None, - gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, - gateway_tls_enabled: false, - endpoint_overrides: &EMPTY_ENDPOINT_OVERRIDES, - } - } - - #[test] - fn podman_config_reads_bind_mount_opt_in_from_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r" -[openshell.drivers.podman] -enable_bind_mounts = true -", - ) - .expect("valid config"); - - let cfg = podman_config_from_context(test_context(Some(&file))).expect("podman config"); - - assert!(cfg.enable_bind_mounts); - } - - #[test] - fn docker_config_reads_bind_mount_opt_in_from_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r" -[openshell.drivers.docker] -enable_bind_mounts = true -", - ) - .expect("valid config"); - - let cfg = docker_config_from_context(test_context(Some(&file))).expect("docker config"); - - assert!(cfg.enable_bind_mounts); - } - - #[test] - fn docker_config_reads_socket_path_from_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r#" -[openshell.drivers.docker] -socket_path = "/tmp/docker.sock" -"#, - ) - .expect("valid config"); - - let cfg = docker_config_from_context(test_context(Some(&file))).expect("docker config"); - - assert_eq!(cfg.socket_path, Some(PathBuf::from("/tmp/docker.sock"))); - } - - #[test] - fn docker_config_reports_selected_invalid_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r" -[openshell.drivers.docker] -unknown_docker_key = true -", - ) - .expect("valid config"); - - let err = docker_config_from_context(test_context(Some(&file))).unwrap_err(); - - assert!( - err.to_string() - .contains("invalid [openshell.drivers.docker] table") - ); - } - - #[test] - fn vm_config_reports_selected_invalid_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r#" -[openshell.drivers.vm] -mem_mib = "not-a-number" -"#, - ) - .expect("valid config"); - - let err = vm_config_from_context(test_context(Some(&file))).unwrap_err(); - - assert!( - err.to_string() - .contains("invalid [openshell.drivers.vm] table") - ); - } -} diff --git a/crates/openshell-server/src/compute/lease.rs b/crates/openshell-server/src/compute/lease.rs index bf58fae48b..3310946dab 100644 --- a/crates/openshell-server/src/compute/lease.rs +++ b/crates/openshell-server/src/compute/lease.rs @@ -242,10 +242,10 @@ impl ReconcilerLease { /// Derive a stable replica identity for lease ownership. /// -/// Kubernetes sets `HOSTNAME` to the pod name, Docker sets it to the -/// container ID, and systemd units inherit the machine hostname. -/// `OPENSHELL_REPLICA_ID` allows explicit override. The UUID fallback -/// handles edge cases where neither env var is set. +/// Managed workloads commonly receive a stable runtime identity through +/// `HOSTNAME`, while systemd units inherit the machine hostname. +/// `OPENSHELL_REPLICA_ID` allows an explicit override. The UUID fallback +/// handles environments where neither variable is set. pub fn replica_id() -> String { std::env::var("OPENSHELL_REPLICA_ID") .or_else(|_| std::env::var("HOSTNAME")) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 78ae6fddb3..8fd15d944c 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -5,17 +5,6 @@ pub mod driver_config; pub mod lease; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub mod vm; - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub use openshell_driver_docker::DockerComputeConfig; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub use openshell_driver_kubernetes::KubernetesComputeConfig; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub use openshell_driver_podman::PodmanComputeConfig; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub use vm::VmComputeConfig; use crate::grpc::policy::SANDBOX_SETTINGS_OBJECT_TYPE; use crate::otel_tracing::TraceContextInterceptor; @@ -30,7 +19,6 @@ use crate::tracing_bus::TracingLogBus; use futures::{Stream, StreamExt}; #[cfg(unix)] use hyper_util::rt::TokioIo; -use openshell_core::ComputeDriverKind; #[cfg(target_os = "windows")] use openshell_core::proto::SandboxPolicy; use openshell_core::proto::compute::v1::{ @@ -52,17 +40,6 @@ use openshell_core::proto::{ SandboxTemplate, ServiceEndpoint, SshSession, }; use openshell_core::{ObjectLabels, ObjectWorkspace}; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -use openshell_driver_docker::{ComputeDriverService as DockerDriverService, DockerComputeDriver}; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -use openshell_driver_kubernetes::{ - ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, - OperatorNamespaceAllowlist, -}; -#[cfg(target_os = "windows")] -use openshell_driver_mxc::{ComputeDriverService as MxcDriverService, MxcComputeConfig}; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; use prost::Message; use std::collections::HashMap; use std::fmt; @@ -306,8 +283,8 @@ pub struct ManagedDriverProcess { } impl ManagedDriverProcess { - #[cfg(all(unix, any(test, feature = "in-tree-compute-drivers")))] - pub(crate) fn new(child: tokio::process::Child, socket_path: PathBuf) -> Self { + #[cfg(unix)] + pub fn new(child: tokio::process::Child, socket_path: PathBuf) -> Self { Self { child: std::sync::Mutex::new(Some(child)), socket_path, @@ -404,14 +381,13 @@ pub struct AcquiredRemoteDriverEndpoint { } impl AcquiredRemoteDriverEndpoint { - #[cfg(any(test, feature = "in-tree-compute-drivers"))] - pub(crate) fn managed_builtin( - driver_kind: ComputeDriverKind, + pub fn managed( + name: impl Into, channel: Channel, driver_process: Arc, ) -> Self { Self { - name: driver_kind.as_str().to_string(), + name: name.into(), channel, driver_process: Some(driver_process), } @@ -631,11 +607,9 @@ impl ComputeRuntime { compute_error_from_status(status) })? .into_inner(); - let driver_kind = driver_name.parse::().ok(); info!( configured_driver = %driver_name, advertised_driver = %capabilities.driver_name, - in_tree = driver_kind.is_some(), "Compute driver connected" ); let driver_info = ComputeDriverInfoSnapshot { @@ -743,62 +717,6 @@ impl ComputeRuntime { self.lifecycle_gates.entry_count() } - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - pub async fn new_docker( - config: openshell_core::Config, - docker_config: DockerComputeConfig, - store: Arc, - sandbox_index: SandboxIndex, - sandbox_watch_bus: SandboxWatchBus, - tracing_log_bus: TracingLogBus, - supervisor_sessions: Arc, - ) -> Result { - let driver = DockerComputeDriver::new(&config, &docker_config) - .await - .map_err(|err| ComputeError::Message(err.to_string()))?; - let driver: SharedComputeDriver = Arc::new(DockerDriverService::new_in_process(driver)); - Self::from_driver( - ComputeDriverKind::Docker.as_str().to_string(), - driver, - None, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - } - - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - pub async fn new_kubernetes( - config: KubernetesComputeConfig, - store: Arc, - sandbox_index: SandboxIndex, - sandbox_watch_bus: SandboxWatchBus, - tracing_log_bus: TracingLogBus, - supervisor_sessions: Arc, - shutdown_rx: watch::Receiver, - ) -> Result<(Self, Option), ComputeError> { - let driver = KubernetesComputeDriver::new(config, shutdown_rx) - .await - .map_err(|err| ComputeError::Message(err.to_string()))?; - let operator_allowlist_arc = driver.operator_allowlist().cloned(); - let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new_in_process(driver)); - let runtime = Self::from_driver( - ComputeDriverKind::Kubernetes.as_str().to_string(), - driver, - None, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await?; - Ok((runtime, operator_allowlist_arc)) - } - pub(crate) async fn new_remote_driver( endpoint: AcquiredRemoteDriverEndpoint, store: Arc, @@ -821,64 +739,6 @@ impl ComputeRuntime { .await } - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - pub async fn new_podman( - config: PodmanComputeConfig, - store: Arc, - sandbox_index: SandboxIndex, - sandbox_watch_bus: SandboxWatchBus, - tracing_log_bus: TracingLogBus, - supervisor_sessions: Arc, - ) -> Result { - let driver = PodmanComputeDriver::new(config) - .await - .map_err(|err| ComputeError::Message(err.to_string()))?; - let driver: SharedComputeDriver = Arc::new(PodmanDriverService::new_in_process(driver)); - Self::from_driver( - ComputeDriverKind::Podman.as_str().to_string(), - driver, - None, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - } - - /// Construct a `ComputeRuntime` backed by the MXC compute driver. - /// - /// MXC is Windows-only, in-process, and self-reports `Ready` — there is - /// no supervisor session argument because no surrogate or relay is used. - #[cfg(target_os = "windows")] - pub async fn new_mxc( - mxc_config: MxcComputeConfig, - store: Arc, - sandbox_index: SandboxIndex, - sandbox_watch_bus: SandboxWatchBus, - tracing_log_bus: TracingLogBus, - supervisor_sessions: Arc, - ) -> Result { - let backend = openshell_driver_mxc::MxcComputeBackend::new(mxc_config); - // Grab the A1 policy side channel before moving `backend` into the service. - let sink = backend.policy_sink(); - let service: SharedComputeDriver = Arc::new(MxcDriverService::new(backend)); - let mut runtime = Self::from_driver( - ComputeDriverKind::Mxc.as_str().to_string(), - service, - None, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await?; - runtime.mxc_policy_sink = Some(sink); - Ok(runtime) - } - #[must_use] pub fn default_image(&self) -> &str { &self.default_image @@ -890,8 +750,8 @@ impl ComputeRuntime { } #[must_use] - pub fn driver_kind(&self) -> Option { - self.driver_info.name.parse().ok() + pub fn configured_driver_name(&self) -> &str { + &self.driver_info.name } #[must_use] @@ -1421,12 +1281,11 @@ impl ComputeRuntime { let runtime_restart_during_stop = expected_stopped && driver_snapshot_reports_runtime_restart(&snapshot); if suspension_progressing || runtime_restart_during_stop { - // The Kubernetes controller has accepted the stop and - // is waiting for its pod to terminate. A container - // runtime restart can likewise be the expected SIGTERM - // exit from an in-flight stop. Preserve the durable - // transition so completion or recovery, rather than the - // watcher, determines its terminal state. + // The backend has accepted the stop but has not finished + // terminating the sandbox. A runtime restart can likewise + // be the expected exit from an in-flight stop. Preserve the + // durable transition so completion or recovery, rather than + // a watch snapshot, determines its terminal state. debug!(sandbox_id, "Sandbox stop is still progressing"); } else if backend_phase == SandboxPhase::Error || observed_stopped == expected_stopped @@ -4451,24 +4310,10 @@ fn derive_phase(status: Option<&DriverSandboxStatus>) -> SandboxPhase { return SandboxPhase::Deleting; } - // `Ready=True` means the sandbox is usable through this gateway and must - // win over a `Suspended=True` condition. Agent Sandbox v1beta1 sets - // `Suspended=True (PodTerminated)` on stop and does not clear it on resume, - // so a resumed CR carries both `Ready=True` and a stale `Suspended=True`. - // Treating any `Suspended=True` as Stopped would pin the resumed sandbox at - // Starting forever (issue #2932). A genuine stop leaves `Ready` unset or - // False, so `Suspended` still resolves to Stopped in that case. - let ready = status.conditions.iter().any(|condition| { - condition.r#type.eq_ignore_ascii_case("Ready") + if status.conditions.iter().any(|condition| { + condition.r#type.eq_ignore_ascii_case("Suspended") && condition.status.eq_ignore_ascii_case("true") - }); - - if !ready - && status.conditions.iter().any(|condition| { - condition.r#type.eq_ignore_ascii_case("Suspended") - && condition.status.eq_ignore_ascii_case("true") - }) - { + }) { return SandboxPhase::Stopped; } @@ -6480,8 +6325,7 @@ mod tests { )); } - /// Driver calls are a remote boundary even in-process: they reach the - /// Docker daemon, the Kubernetes API, or a Podman socket. + /// Driver calls are a remote boundary even when the driver is in-process. #[tokio::test] async fn driver_calls_export_spans_with_parents() { use tracing::Instrument as _; @@ -7371,56 +7215,6 @@ mod tests { assert_eq!(current.phase(), SandboxPhase::Stopped as i32); } - #[tokio::test] - async fn resumed_v1beta1_snapshot_with_stale_suspended_reaches_ready() { - // Reproduces issue #2932: on Agent Sandbox v1beta1 a resumed CR reports - // Ready=True (DependenciesReady) alongside a stale Suspended=True - // (PodTerminated). Starting from the Starting phase that `start` sets, the - // reconciled sandbox must advance to Ready rather than being pinned at - // Starting by the stale Suspended condition. - let runtime = test_runtime(Arc::new(TestDriver::default())).await; - let sandbox = sandbox_record("sb-resumed", "sandbox-resumed", SandboxPhase::Starting); - runtime.store.put_message(&sandbox).await.unwrap(); - register_test_supervisor_session(&runtime, sandbox.object_id()); - - let mut resumed = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name()); - resumed.status = Some(DriverSandboxStatus { - sandbox_name: sandbox.object_name().to_string(), - instance_id: format!("{}-pod", sandbox.object_name()), - conditions: vec![ - DriverCondition { - r#type: "Ready".to_string(), - status: "True".to_string(), - reason: "DependenciesReady".to_string(), - message: "Sandbox is ready".to_string(), - last_transition_time: String::new(), - }, - DriverCondition { - r#type: "Suspended".to_string(), - status: "True".to_string(), - reason: "PodTerminated".to_string(), - message: "Pod terminated".to_string(), - last_transition_time: String::new(), - }, - ], - ..Default::default() - }); - - runtime.apply_sandbox_update(resumed).await.unwrap(); - - let current = runtime - .store - .get_message::(sandbox.object_id()) - .await - .unwrap() - .unwrap(); - assert_eq!( - current.phase(), - SandboxPhase::Ready as i32, - "a resumed, Ready sandbox must not stay Starting because of a stale Suspended condition" - ); - } - #[tokio::test] async fn stopped_container_snapshot_cannot_error_stopped_sandbox() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; @@ -9239,7 +9033,7 @@ mod tests { #[tokio::test] async fn backend_not_ready_with_supervisor_becomes_ready() { - // VM path: supervisor connects before backend reports Ready. + // The supervisor may connect before the backend reports Ready. let runtime = test_runtime(Arc::new(TestDriver::default())).await; let sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Provisioning); runtime.store.put_message(&sandbox).await.unwrap(); diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 74b6aad01b..65709748b1 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -26,7 +26,6 @@ use std::net::SocketAddr; use std::path::{Path, PathBuf}; use base64::Engine as _; -use openshell_core::config::ComputeDriverKind; use openshell_core::proto::SupervisorMiddlewareService; use openshell_core::{ GatewayAuthConfig, GatewayInterceptorConfig, GatewayJwtConfig, @@ -432,13 +431,22 @@ pub fn driver_table( driver_name: &str, gateway: &GatewayFileSection, raw: Option<&toml::Value>, +) -> toml::Value { + driver_table_with_inherited_keys(driver_name, gateway, raw, &[]) +} + +pub(crate) fn driver_table_with_inherited_keys( + _driver_name: &str, + gateway: &GatewayFileSection, + raw: Option<&toml::Value>, + inheritable_keys: &[&str], ) -> toml::Value { let mut merged = match raw { Some(toml::Value::Table(table)) => table.clone(), _ => toml::Table::new(), }; - for key in inheritable_keys(driver_name) { + for key in inheritable_keys { if merged.contains_key(*key) { continue; } @@ -450,50 +458,6 @@ pub fn driver_table( toml::Value::Table(merged) } -/// Inheritance allowlist (the Q4 "high-overlap set"). Each driver opts in -/// to a specific subset so a gateway-wide default does not accidentally land -/// in a driver table that does not understand the field. -fn inheritable_keys(driver_name: &str) -> &'static [&'static str] { - match driver_name.parse::().ok() { - Some(ComputeDriverKind::Kubernetes) => &[ - "namespace", - "default_image", - "supervisor_image", - "client_tls_secret_name", - "service_account_name", - "host_gateway_ip", - "enable_user_namespaces", - "sa_token_ttl_secs", - ], - Some(ComputeDriverKind::Docker) => &[ - "sandbox_namespace", - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ], - Some(ComputeDriverKind::Podman) => &[ - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ], - Some(ComputeDriverKind::Vm) => &[ - "default_image", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ], - // MXC reads its own settings from the driver config table and has no - // gateway-inherited required fields. - Some(ComputeDriverKind::Mxc) | None => &[], - } -} - fn gateway_inherited_value(g: &GatewayFileSection, key: &str) -> Option { match key { "namespace" | "sandbox_namespace" => g.sandbox_namespace.as_deref().map(string_value), @@ -986,10 +950,11 @@ version = 2 let raw = toml::toml! { namespace = "agents" }; - let merged = driver_table( - ComputeDriverKind::Kubernetes.as_str(), + let merged = driver_table_with_inherited_keys( + "alpha", &gateway, Some(&toml::Value::Table(raw)), + &["default_image", "supervisor_image"], ); let table = merged.as_table().expect("table"); assert_eq!( @@ -1007,14 +972,19 @@ version = 2 } #[test] - fn docker_driver_table_inherits_gateway_defaults() { + fn registered_driver_table_inherits_selected_gateway_defaults() { let gateway = GatewayFileSection { sandbox_namespace: Some("agents".to_string()), default_image: Some("ghcr.io/nvidia/openshell/sandbox:0.9".to_string()), host_gateway_ip: Some("10.0.0.1".to_string()), ..Default::default() }; - let merged = driver_table(ComputeDriverKind::Docker.as_str(), &gateway, None); + let merged = driver_table_with_inherited_keys( + "alpha", + &gateway, + None, + &["sandbox_namespace", "default_image", "host_gateway_ip"], + ); let table = merged.as_table().expect("table"); assert_eq!( table.get("sandbox_namespace").and_then(|v| v.as_str()), @@ -1031,13 +1001,18 @@ version = 2 } #[test] - fn podman_driver_table_inherits_gateway_host_gateway_ip() { + fn registered_driver_table_can_select_network_defaults() { let gateway = GatewayFileSection { default_image: Some("ghcr.io/nvidia/openshell/sandbox:0.9".to_string()), host_gateway_ip: Some("192.168.127.254".to_string()), ..Default::default() }; - let merged = driver_table(ComputeDriverKind::Podman.as_str(), &gateway, None); + let merged = driver_table_with_inherited_keys( + "beta", + &gateway, + None, + &["default_image", "host_gateway_ip"], + ); let table = merged.as_table().expect("table"); assert_eq!( table.get("default_image").and_then(|v| v.as_str()), @@ -1058,10 +1033,11 @@ version = 2 let raw = toml::toml! { default_image = "driver-specific" }; - let merged = driver_table( - ComputeDriverKind::Podman.as_str(), + let merged = driver_table_with_inherited_keys( + "alpha", &gateway, Some(&toml::Value::Table(raw)), + &["default_image"], ); assert_eq!( merged @@ -1075,13 +1051,12 @@ version = 2 #[test] fn driver_table_does_not_leak_keys_outside_allowlist() { - // `client_tls_secret_name` is K8s-only; Docker must not receive it - // even when set at gateway scope. + // Fields not selected by the registration must remain gateway-only. let gateway = GatewayFileSection { client_tls_secret_name: Some("openshell-sandbox-tls".to_string()), ..Default::default() }; - let merged = driver_table(ComputeDriverKind::Docker.as_str(), &gateway, None); + let merged = driver_table_with_inherited_keys("alpha", &gateway, None, &["default_image"]); assert!( !merged .as_table() diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index 640757bfb3..b638fc33e4 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -386,10 +386,10 @@ mod tests { #[test] fn gateway_listener_specs_reuse_primary_when_wildcard_covers_driver_address() { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); - let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); + let callback: SocketAddr = "172.18.0.1:8080".parse().unwrap(); let requirements = [ - docker_listener_requirement(docker), - docker_listener_requirement(docker), + exact_listener_requirement(callback), + exact_listener_requirement(callback), ]; assert_eq!( @@ -401,15 +401,15 @@ mod tests { #[test] fn gateway_listener_scope_for_reused_primary_remains_primary() { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); - let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); + let callback: SocketAddr = "172.18.0.1:8080".parse().unwrap(); let loopback: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let [spec] = gateway_listener_specs(primary, &[docker_listener_requirement(docker)]) + let [spec] = gateway_listener_specs(primary, &[exact_listener_requirement(callback)]) .unwrap() .try_into() .unwrap(); assert_eq!( - spec.scope_for_local_addr(docker), + spec.scope_for_local_addr(callback), GatewayListenerScope::Primary, ); assert_eq!( @@ -421,10 +421,10 @@ mod tests { #[test] fn gateway_listener_specs_preserve_driver_callback_scope() { let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); + let callback: SocketAddr = "172.18.0.1:8080".parse().unwrap(); let requirements = [ - docker_listener_requirement(docker), - docker_listener_requirement(docker), + exact_listener_requirement(callback), + exact_listener_requirement(callback), ]; assert_eq!( @@ -437,11 +437,11 @@ mod tests { provenance: None, }, GatewayListenerSpec { - address: docker, + address: callback, scope: GatewayListenerScope::ComputeDriverCallback, covered_addresses: Vec::new(), provenance: Some(GatewayListenerProvenance { - driver_name: "docker".to_string(), + driver_name: "alpha".to_string(), reason: "managed bridge".to_string(), }), }, @@ -473,7 +473,7 @@ mod tests { "172.18.0.1:0", "172.18.0.1:9090", ] { - let requirement = docker_listener_requirement(address.parse().unwrap()); + let requirement = exact_listener_requirement(address.parse().unwrap()); assert!( gateway_listener_specs(primary, &[requirement]).is_err(), "{address} should be rejected" @@ -482,41 +482,41 @@ mod tests { } #[test] - fn gateway_listener_specs_use_exact_podman_network_gateway() { + fn gateway_listener_specs_use_exact_network_gateway() { let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let podman_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); + let network_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs(primary, &[podman_listener_requirement(podman_gateway)]) + gateway_listener_specs(primary, &[network_listener_requirement(network_gateway)]) .unwrap(), vec![ primary_listener_spec(primary), - callback_listener_spec(podman_gateway, "podman", "Podman managed bridge",), + callback_listener_spec(network_gateway, "beta", "managed bridge",), ] ); } #[test] - fn gateway_listener_specs_reuse_primary_when_it_covers_podman_exact() { + fn gateway_listener_specs_reuse_primary_when_it_covers_exact_requirement() { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); - let podman_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); + let network_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs(primary, &[podman_listener_requirement(podman_gateway)],) + gateway_listener_specs(primary, &[network_listener_requirement(network_gateway)],) .unwrap(), vec![primary_listener_spec(primary)] ); } #[test] - fn gateway_listener_specs_resolve_podman_default_route_source() { + fn gateway_listener_specs_resolve_default_route_source() { let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); let default_route_ip = "192.168.20.20".parse().unwrap(); assert_eq!( gateway_listener_specs_with_default_route_ip( primary, - &[podman_default_route_listener_requirement()], + &[default_route_listener_requirement()], Some(default_route_ip), ) .unwrap(), @@ -524,8 +524,8 @@ mod tests { primary_listener_spec(primary), callback_listener_spec( "192.168.20.20:8080".parse().unwrap(), - "podman", - "rootless pasta upstream interface", + "beta", + "default route interface", ), ] ); @@ -537,7 +537,7 @@ mod tests { let err = gateway_listener_specs_with_default_route_ip( primary, - &[podman_default_route_listener_requirement()], + &[default_route_listener_requirement()], Some("203.0.113.20".parse().unwrap()), ) .unwrap_err(); @@ -552,7 +552,7 @@ mod tests { assert_eq!( gateway_listener_specs_with_default_route_ip( primary, - &[podman_default_route_listener_requirement()], + &[default_route_listener_requirement()], Some(default_route_ip), ) .unwrap(), @@ -561,28 +561,24 @@ mod tests { } #[test] - fn gateway_listener_specs_resolve_podman_loopback_separately() { + fn gateway_listener_specs_resolve_loopback_separately() { let primary: SocketAddr = "192.168.20.20:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(), vec![ primary_listener_spec(primary), - callback_listener_spec( - "127.0.0.1:8080".parse().unwrap(), - "podman", - "Podman machine host forwarder", - ), + callback_listener_spec("127.0.0.1:8080".parse().unwrap(), "beta", "host forwarder",), ] ); } #[test] - fn gateway_listener_specs_reuse_wildcard_primary_for_podman_loopback() { + fn gateway_listener_specs_reuse_wildcard_primary_for_loopback() { let primary = "0.0.0.0:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(), vec![primary_listener_spec(primary)] ); } @@ -592,7 +588,7 @@ mod tests { let primary = "127.0.0.1:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(), vec![primary_listener_spec(primary)] ); } @@ -602,7 +598,7 @@ mod tests { for primary in ["[::1]:8080", "[::]:8080"] { let primary = primary.parse().unwrap(); let specs = - gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(); + gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(); assert_eq!(specs.len(), 2); assert_eq!(specs[1].address, SocketAddr::from(([127, 0, 0, 1], 8080))); @@ -613,7 +609,7 @@ mod tests { fn gateway_listener_specs_validate_selector_independently_of_driver_name() { let primary: SocketAddr = "192.168.20.20:8080".parse().unwrap(); let requirement = GatewayListenerRequirement::LoopbackInterface { - driver_name: "docker".to_string(), + driver_name: "alpha".to_string(), reason: "wrong selector".to_string(), }; @@ -634,7 +630,7 @@ mod tests { let result: openshell_core::Result<()> = async { let _listeners = bind_gateway_listeners( primary_address, - &[docker_listener_requirement(occupied_address)], + &[exact_listener_requirement(occupied_address)], ) .await?; continuation_reached.store(true, Ordering::SeqCst); @@ -663,7 +659,7 @@ mod tests { drop(probe); let primary = format!("[::]:{port}").parse().unwrap(); - let listeners = bind_gateway_listeners(primary, &[podman_loopback_listener_requirement()]) + let listeners = bind_gateway_listeners(primary, &[loopback_listener_requirement()]) .await .expect("IPv6 wildcard and IPv4 callback listeners should both bind"); @@ -675,33 +671,33 @@ mod tests { ); } - fn docker_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { + fn exact_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { GatewayListenerRequirement::Exact { address, - driver_name: "docker".to_string(), + driver_name: "alpha".to_string(), reason: "managed bridge".to_string(), } } - fn podman_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { + fn network_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { GatewayListenerRequirement::Exact { address, - driver_name: "podman".to_string(), - reason: "Podman managed bridge".to_string(), + driver_name: "beta".to_string(), + reason: "managed bridge".to_string(), } } - fn podman_default_route_listener_requirement() -> GatewayListenerRequirement { + fn default_route_listener_requirement() -> GatewayListenerRequirement { GatewayListenerRequirement::DefaultRouteInterface { - driver_name: "podman".to_string(), - reason: "rootless pasta upstream interface".to_string(), + driver_name: "beta".to_string(), + reason: "default route interface".to_string(), } } - fn podman_loopback_listener_requirement() -> GatewayListenerRequirement { + fn loopback_listener_requirement() -> GatewayListenerRequirement { GatewayListenerRequirement::LoopbackInterface { - driver_name: "podman".to_string(), - reason: "Podman machine host forwarder".to_string(), + driver_name: "beta".to_string(), + reason: "host forwarder".to_string(), } } diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index de658909b0..227db094a0 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -172,7 +172,7 @@ fn emit_sandbox_create_telemetry( request: &CreateSandboxRequest, outcome: TelemetryOutcome, ) { - let compute_driver = telemetry_compute_driver(state.compute.driver_kind()); + let compute_driver = telemetry_compute_driver(state.compute.configured_driver_name()); let Some(spec) = request.spec.as_ref() else { openshell_core::telemetry::emit_sandbox_create( outcome, @@ -205,10 +205,8 @@ fn emit_sandbox_create_telemetry( ); } -fn telemetry_compute_driver( - driver_kind: Option, -) -> TelemetryComputeDriver { - TelemetryComputeDriver::from_driver_kind(driver_kind) +fn telemetry_compute_driver(driver_name: &str) -> TelemetryComputeDriver { + TelemetryComputeDriver::from_raw(driver_name) } async fn handle_create_sandbox_inner( @@ -347,11 +345,8 @@ async fn handle_create_sandbox_inner( status })?; - // Mint the gateway JWT for singleplayer drivers. K8s sandboxes skip - // this mint and bootstrap via `IssueSandboxToken` at supervisor - // startup; identifying "is this K8s?" lives in the compute layer, so - // we mint unconditionally here when the issuer is configured and let - // the K8s driver simply ignore the field. + // Mint a gateway JWT whenever the issuer is configured. Compute runtimes + // that bootstrap through another authentication mechanism may ignore it. let sandbox_token = state.sandbox_jwt_issuer.as_ref().map(|issuer| { issuer.mint(&id).map(|minted| { tracing::info!( @@ -2603,24 +2598,24 @@ mod tests { #[test] fn telemetry_compute_driver_uses_resolved_driver_kind() { assert_eq!( - telemetry_compute_driver(Some(openshell_core::ComputeDriverKind::Docker)), - TelemetryComputeDriver::Docker + telemetry_compute_driver("docker"), + TelemetryComputeDriver::from_raw("docker") ); assert_eq!( - telemetry_compute_driver(Some(openshell_core::ComputeDriverKind::Kubernetes)), - TelemetryComputeDriver::Kubernetes + telemetry_compute_driver("kubernetes"), + TelemetryComputeDriver::from_raw("kubernetes") ); assert_eq!( - telemetry_compute_driver(Some(openshell_core::ComputeDriverKind::Podman)), - TelemetryComputeDriver::Podman + telemetry_compute_driver("podman"), + TelemetryComputeDriver::from_raw("podman") ); assert_eq!( - telemetry_compute_driver(Some(openshell_core::ComputeDriverKind::Vm)), - TelemetryComputeDriver::Vm + telemetry_compute_driver("vm"), + TelemetryComputeDriver::from_raw("vm") ); assert_eq!( - telemetry_compute_driver(None), - TelemetryComputeDriver::Unknown + telemetry_compute_driver(""), + TelemetryComputeDriver::from_raw("") ); } @@ -3636,8 +3631,7 @@ mod tests { #[tokio::test] async fn create_and_get_preserve_partial_process_identity() { - let state = - test_server_state_with_driver(openshell_core::ComputeDriverKind::Docker.as_str()).await; + let state = test_server_state_with_driver("docker").await; let policy = openshell_core::proto::SandboxPolicy { version: 1, process: Some(openshell_core::proto::ProcessPolicy { @@ -3701,9 +3695,7 @@ mod tests { #[tokio::test] async fn create_and_get_preserve_partial_process_identity_for_kubernetes() { - let state = - test_server_state_with_driver(openshell_core::ComputeDriverKind::Kubernetes.as_str()) - .await; + let state = test_server_state_with_driver("kubernetes").await; let policy = openshell_core::proto::SandboxPolicy { version: 1, process: Some(openshell_core::proto::ProcessPolicy { diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index c2306b6632..3c1c5c64dc 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -13,16 +13,6 @@ //! startup. Runtime selection only consults that registry or a configured //! external endpoint; it does not switch on driver names. -// `defaults-without-telemetry` is an alias for the default feature set minus -// `telemetry`, not a switch that turns telemetry off. Cargo cannot subtract a -// default feature, so adding it on top of the defaults would otherwise produce -// a telemetry-on build that reads as telemetry-free. Fail the build instead. -#[cfg(all(feature = "telemetry", feature = "defaults-without-telemetry"))] -compile_error!( - "features `telemetry` and `defaults-without-telemetry` are mutually exclusive; \ - build a telemetry-free gateway with `--no-default-features --features defaults-without-telemetry`" -); - mod auth; pub mod certgen; pub mod cli; @@ -58,8 +48,6 @@ mod tracing_setup; mod ws_tunnel; use metrics_exporter_prometheus::PrometheusBuilder; -#[cfg(target_os = "windows")] -use openshell_core::ComputeDriverKind; use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::{Config, Error, ObjectLabels, Result}; use openshell_extension_core::{ @@ -69,7 +57,7 @@ use openshell_supervisor_middleware::MiddlewareRegistry; use std::collections::{BTreeMap, HashMap}; use std::io::ErrorKind; use std::net::SocketAddr; -use std::path::Path; +use std::path::{Path, PathBuf}; #[cfg(test)] use std::sync::LazyLock; use std::sync::{ @@ -303,8 +291,8 @@ pub struct ServerState { /// Registry of active supervisor sessions and pending relay channels. /// - /// Stored as `Arc` so compute drivers (e.g. the Docker driver) - /// can be constructed before `ServerState` and still + /// Stored as `Arc` so compiled compute drivers can be constructed before + /// `ServerState` and still /// query session state to surface supervisor readiness. pub supervisor_sessions: Arc, @@ -452,14 +440,14 @@ impl ServerState { /// Returns an error if the server fails to start or encounters a fatal error. pub(crate) async fn run_server( startup: ServerStartupConfig, - compute_driver: ConfiguredComputeDriver, tracing_log_bus: TracingLogBus, + compute_drivers: ComputeDriverRegistry, ) -> Result<()> { let ServerStartupConfig { config, config_file, guest_tls, - compute_driver: _, + compute_driver, } = startup; let (shutdown_tx, shutdown_rx) = watch::channel(false); @@ -600,12 +588,18 @@ pub(crate) async fn run_server( let sandbox_index = SandboxIndex::new(); let sandbox_watch_bus = SandboxWatchBus::new(); let supervisor_sessions = Arc::new(supervisor_session::SupervisorSessionRegistry::new()); - let driver_startup = - compute_driver_startup_context(&config, config_file.as_ref(), guest_tls.as_ref()); - let (compute, _operator_allowlist) = build_compute_runtime( + let driver_startup = compute::driver_config::DriverStartupContext { + file: config_file.as_ref(), + guest_tls: guest_tls.as_ref(), + gateway_port: config.bind_address.port(), + gateway_tls_enabled: config.tls.is_some(), + endpoint_overrides: &config.compute_driver_endpoints, + }; + let compute = build_compute_runtime( + &compute_drivers, + &compute_driver, &config, driver_startup, - compute_driver, store.clone(), sandbox_index.clone(), sandbox_watch_bus.clone(), @@ -1048,30 +1042,59 @@ async fn terminate_signal() { let _ = signal.recv().await; } -#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] -fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::ComputeError { - compute::ComputeError::Message(format!( - "{} compute driver is unsupported on Windows", - driver.as_str() - )) +pub use compute::{ + AcquiredRemoteDriverEndpoint, DriverWatchStream, ManagedDriverProcess, SharedComputeDriver, +}; + +/// Driver instance returned by a compiled compute-driver factory. +pub enum ComputeDriverInstance { + /// A driver hosted in the gateway process. + InProcess(SharedComputeDriver), + /// A driver process launched and owned by the gateway. + ManagedRemote(AcquiredRemoteDriverEndpoint), } -type OperatorAllowlistArc = Option; -pub use compute::{DriverWatchStream, SharedComputeDriver}; +/// Type-erased tracing layer contributed by a compiled compute driver. +pub type ComputeDriverTracingLayer = + Box + Send + Sync>; + +/// Shutdown callback for resources owned by a compute-driver tracing layer. +pub type ComputeDriverTracingShutdown = + Box std::result::Result<(), String> + Send + Sync>; + +/// Optional process-wide tracing integration supplied by a compiled driver. +#[derive(Default)] +pub struct ComputeDriverTracingSetup { + layer: Option, + shutdown: Option, + error: Option, + target_prefix: Option<&'static str>, +} -/// Opaque result returned by a compiled compute-driver factory. -pub struct ComputeDriverBuildOutput { - runtime: ComputeRuntime, - operator_allowlist: OperatorAllowlistArc, +impl ComputeDriverTracingSetup { + #[must_use] + pub fn new( + layer: Option, + shutdown: Option, + error: Option, + target_prefix: Option<&'static str>, + ) -> Self { + Self { + layer, + shutdown, + error, + target_prefix, + } + } } +/// Factory for a compiled driver's optional tracing integration. +pub type ComputeDriverTracingFactory = fn(Option<&str>) -> ComputeDriverTracingSetup; + /// Factory for a compute driver linked into a gateway binary. #[async_trait::async_trait] pub trait ComputeDriverFactory: Send + Sync { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result; + async fn build(&self, context: ComputeDriverBuildContext<'_>) -> Result; } /// One named compiled-driver registration. @@ -1081,6 +1104,10 @@ pub struct ComputeDriverRegistration { detection_priority: u16, detect: Option bool>, factory: Arc, + inherited_config_keys: &'static [&'static str], + local_singleplayer: bool, + supports_mtls_user_auth: bool, + tracing_setup: Option, } impl std::fmt::Debug for ComputeDriverRegistration { @@ -1109,8 +1136,50 @@ impl ComputeDriverRegistration { detection_priority, detect, factory: Arc::new(factory), + inherited_config_keys: &[], + local_singleplayer: false, + supports_mtls_user_auth: true, + tracing_setup: None, }) } + + /// Select gateway-wide defaults understood by this driver's config type. + #[must_use] + pub fn with_inherited_config_keys(mut self, keys: &'static [&'static str]) -> Self { + self.inherited_config_keys = keys; + self + } + + /// Mark a backend whose local deployment should use single-player defaults. + #[must_use] + pub fn with_local_singleplayer(mut self) -> Self { + self.local_singleplayer = true; + self + } + + /// Mark a backend that requires user authentication other than mTLS. + #[must_use] + pub fn without_mtls_user_auth(mut self) -> Self { + self.supports_mtls_user_auth = false; + self + } + + /// Attach optional process-wide tracing for this compiled driver. + #[must_use] + pub fn with_tracing_setup(mut self, setup: ComputeDriverTracingFactory) -> Self { + self.tracing_setup = Some(setup); + self + } + + #[must_use] + pub(crate) fn is_local_singleplayer(&self) -> bool { + self.local_singleplayer + } + + #[must_use] + pub(crate) fn supports_mtls_user_auth(&self) -> bool { + self.supports_mtls_user_auth + } } /// Registry of compute drivers compiled into this gateway binary. @@ -1175,10 +1244,27 @@ impl ComputeDriverRegistry { self.drivers.keys().map(String::as_str) } - fn get(&self, name: &str) -> Option<&ComputeDriverRegistration> { + pub(crate) fn get(&self, name: &str) -> Option<&ComputeDriverRegistration> { self.drivers.get(name) } + fn tracing_setup( + &self, + selection: &ComputeDriverSelection, + endpoint_overrides: &BTreeMap, + otlp_endpoint: Option<&str>, + ) -> ComputeDriverTracingSetup { + let name = selection.name(); + if endpoint_overrides.contains_key(name) { + return ComputeDriverTracingSetup::default(); + } + self.get(name) + .and_then(|registration| registration.tracing_setup) + .map_or_else(ComputeDriverTracingSetup::default, |setup| { + setup(otlp_endpoint) + }) + } + fn detect(&self) -> ComputeDriverDetection { let mut candidates = self .drivers @@ -1205,7 +1291,7 @@ impl ComputeDriverRegistry { if detection.selected().is_none() { return Err(Error::config( "no compute driver configured and auto-detection found no suitable installed \ - driver; set --drivers or OPENSHELL_DRIVERS=", + driver; set --drivers or OPENSHELL_DRIVERS=", )); } Ok(ComputeDriverSelection::AutoDetected(detection)) @@ -1223,88 +1309,13 @@ impl ComputeDriverRegistry { } } -/// Install every first-party compute driver linked into the standard gateway. -#[must_use] -pub fn install_default_compute_drivers() -> ComputeDriverRegistry { - #[allow(unused_mut)] - let mut registry = ComputeDriverRegistry::new(); - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - { - registry - .install( - ComputeDriverRegistration::new( - "kubernetes", - 100, - Some(|| std::env::var_os("KUBERNETES_SERVICE_HOST").is_some()), - KubernetesComputeDriverFactory, - ) - .expect("valid kubernetes registration"), - ) - .expect("unique kubernetes registration"); - registry - .install( - ComputeDriverRegistration::new( - "podman", - 200, - Some(openshell_core::config::is_podman_available), - PodmanComputeDriverFactory, - ) - .expect("valid podman registration"), - ) - .expect("unique podman registration"); - registry - .install( - ComputeDriverRegistration::new( - "docker", - 300, - Some(openshell_core::config::is_docker_available), - DockerComputeDriverFactory, - ) - .expect("valid docker registration"), - ) - .expect("unique docker registration"); - registry - .install( - ComputeDriverRegistration::new("vm", u16::MAX, None, VmComputeDriverFactory) - .expect("valid vm registration"), - ) - .expect("unique vm registration"); - } - #[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] - { - registry - .install( - ComputeDriverRegistration::new("mxc", u16::MAX, None, MxcComputeDriverFactory) - .expect("valid mxc registration"), - ) - .expect("unique mxc registration"); - for name in ["kubernetes", "podman", "docker", "vm"] { - registry - .install( - ComputeDriverRegistration::new( - name, - u16::MAX, - None, - UnsupportedComputeDriverFactory, - ) - .expect("valid unsupported registration"), - ) - .expect("unique unsupported registration"); - } - } - registry -} - pub struct ComputeDriverBuildContext<'a> { driver_name: String, - config: &'a Config, + gateway_bind_address: SocketAddr, + gateway_log_level: &'a str, driver_startup: compute::driver_config::DriverStartupContext<'a>, - store: Arc, - sandbox_index: SandboxIndex, - sandbox_watch_bus: SandboxWatchBus, - tracing_log_bus: TracingLogBus, - supervisor_sessions: Arc, shutdown_rx: watch::Receiver, + inherited_config_keys: &'static [&'static str], } impl ComputeDriverBuildContext<'_> { @@ -1314,8 +1325,13 @@ impl ComputeDriverBuildContext<'_> { } #[must_use] - pub fn gateway_config(&self) -> &Config { - self.config + pub fn gateway_bind_address(&self) -> SocketAddr { + self.gateway_bind_address + } + + #[must_use] + pub fn gateway_log_level(&self) -> &str { + self.gateway_log_level } #[must_use] @@ -1341,7 +1357,11 @@ impl ComputeDriverBuildContext<'_> { where T: Default + serde::de::DeserializeOwned, { - compute::driver_config::driver_config_from_context(self.driver_startup, &self.driver_name) + compute::driver_config::driver_config_from_context( + self.driver_startup, + &self.driver_name, + self.inherited_config_keys, + ) } #[must_use] @@ -1349,242 +1369,84 @@ impl ComputeDriverBuildContext<'_> { self.shutdown_rx.clone() } - /// Finish construction of an in-process driver through the common runtime path. - pub async fn finish_in_process( - self, - driver: SharedComputeDriver, - ) -> Result { - let runtime = ComputeRuntime::from_driver( - self.driver_name, - driver, - None, - self.store, - self.sandbox_index, - self.sandbox_watch_bus, - self.tracing_log_bus, - self.supervisor_sessions, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist: None, - }) - } -} - -#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct MxcComputeDriverFactory; - -#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for MxcComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result { - let mxc_config = compute::driver_config::mxc_config_from_context(context.driver_startup)?; - let runtime = ComputeRuntime::new_mxc( - mxc_config, - context.store, - context.sandbox_index, - context.sandbox_watch_bus, - context.tracing_log_bus, - context.supervisor_sessions, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist: None, - }) - } -} - -#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct UnsupportedComputeDriverFactory; - -#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for UnsupportedComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result { - Err(Error::execution( - unsupported_builtin_compute_driver( - context - .driver_name - .parse() - .expect("default driver names are valid"), - ) - .to_string(), - )) - } -} - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct KubernetesComputeDriverFactory; - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for KubernetesComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result { - warn_if_kubernetes_sandbox_jwt_expiry_disabled(context.config); - let config = compute::driver_config::builtin::kubernetes_config_from_context( - context.driver_startup, - )?; - let (runtime, operator_allowlist) = ComputeRuntime::new_kubernetes( - config, - context.store, - context.sandbox_index, - context.sandbox_watch_bus, - context.tracing_log_bus, - context.supervisor_sessions, - context.shutdown_rx, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist, - }) - } -} - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct DockerComputeDriverFactory; - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for DockerComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result { - let driver_config = - compute::driver_config::builtin::docker_config_from_context(context.driver_startup)?; - let runtime = ComputeRuntime::new_docker( - context.config.clone(), - driver_config, - context.store, - context.sandbox_index, - context.sandbox_watch_bus, - context.tracing_log_bus, - context.supervisor_sessions, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist: None, - }) - } -} - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct PodmanComputeDriverFactory; - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for PodmanComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result { - let driver_config = - compute::driver_config::builtin::podman_config_from_context(context.driver_startup)?; - let runtime = ComputeRuntime::new_podman( - driver_config, - context.store, - context.sandbox_index, - context.sandbox_watch_bus, - context.tracing_log_bus, - context.supervisor_sessions, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist: None, - }) - } -} - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct VmComputeDriverFactory; - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for VmComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result { - let driver_config = - compute::driver_config::builtin::vm_config_from_context(context.driver_startup)?; - let otlp_config = context - .driver_startup + #[must_use] + pub fn otlp_config(&self) -> Option<&config_file::OtlpConfig> { + self.driver_startup .file - .and_then(|file| file.openshell.gateway.otlp.as_ref()); - let endpoint = compute::vm::spawn(context.config, &driver_config, otlp_config).await?; - let runtime = ComputeRuntime::new_remote_driver( - endpoint, - context.store, - context.sandbox_index, - context.sandbox_watch_bus, - context.tracing_log_bus, - context.supervisor_sessions, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist: None, - }) + .and_then(|file| file.openshell.gateway.otlp.as_ref()) } } #[allow(clippy::too_many_arguments)] async fn build_compute_runtime( + registry: &ComputeDriverRegistry, + selection: &ComputeDriverSelection, config: &Config, driver_startup: compute::driver_config::DriverStartupContext<'_>, - driver: ConfiguredComputeDriver, store: Arc, sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, shutdown_rx: watch::Receiver, -) -> Result<(ComputeRuntime, OperatorAllowlistArc)> { +) -> Result { + let driver = resolve_configured_compute_driver(registry, selection.name(), driver_startup)?; info!(driver = %driver.name(), "Using compute driver"); + if config + .gateway_jwt + .as_ref() + .is_some_and(|jwt| jwt.ttl_secs == 0) + && !driver.is_local_singleplayer(registry) + { + warn!( + "Gateway configured with non-expiring sandbox JWTs; set gateway_jwt.ttl_secs > 0 for shared deployments" + ); + } - let (runtime, operator_allowlist) = match driver { + let runtime = match driver { ConfiguredComputeDriver::Registered(registration) => { - let output = registration + let instance = registration .factory .build(ComputeDriverBuildContext { - driver_name: registration.name, - config, + driver_name: registration.name.clone(), + gateway_bind_address: config.bind_address, + gateway_log_level: &config.log_level, driver_startup, + shutdown_rx, + inherited_config_keys: registration.inherited_config_keys, + }) + .await?; + match instance { + ComputeDriverInstance::InProcess(driver) => ComputeRuntime::from_driver( + registration.name, + driver, + None, store, sandbox_index, sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - shutdown_rx, - }) - .await?; - (output.runtime, output.operator_allowlist) + ) + .await + .map_err(|error| { + Error::execution(format!("failed to create compute runtime: {error}")) + })?, + ComputeDriverInstance::ManagedRemote(mut endpoint) => { + endpoint.name = registration.name; + ComputeRuntime::new_remote_driver( + endpoint, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions, + ) + .await + .map_err(|error| { + Error::execution(format!("failed to create compute runtime: {error}")) + })? + } + } } ConfiguredComputeDriver::Remote { name } => { let remote_config = @@ -1597,7 +1459,7 @@ async fn build_compute_runtime( let endpoint = compute::connect_remote_compute_driver(name, &remote_config.socket_path) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - let rt = ComputeRuntime::new_remote_driver( + ComputeRuntime::new_remote_driver( endpoint, store, sandbox_index, @@ -1606,58 +1468,45 @@ async fn build_compute_runtime( supervisor_sessions, ) .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - (rt, None) + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))? } }; - Ok((runtime, operator_allowlist)) -} - -fn compute_driver_startup_context<'a>( - config: &'a Config, - config_file: Option<&'a config_file::ConfigFile>, - guest_tls: Option<&'a compute::driver_config::GuestTlsPaths>, -) -> compute::driver_config::DriverStartupContext<'a> { - compute::driver_config::DriverStartupContext { - file: config_file, - guest_tls, - gateway_port: config.bind_address.port(), - gateway_tls_enabled: config.tls.is_some(), - endpoint_overrides: &config.compute_driver_endpoints, - } + Ok(runtime) } #[derive(Debug, Clone)] -pub(crate) enum ConfiguredComputeDriver { +enum ConfiguredComputeDriver { Registered(ComputeDriverRegistration), Remote { name: String }, } impl ConfiguredComputeDriver { - pub(crate) fn name(&self) -> &str { + fn name(&self) -> &str { match self { Self::Registered(registration) => ®istration.name, Self::Remote { name } => name, } } + + fn is_local_singleplayer(&self, registry: &ComputeDriverRegistry) -> bool { + match self { + Self::Registered(registration) => registration.is_local_singleplayer(), + Self::Remote { name } => registry + .get(name) + .is_some_and(ComputeDriverRegistration::is_local_singleplayer), + } + } } -pub(crate) fn configured_compute_driver_for_startup( +#[cfg(test)] +fn configured_compute_driver( registry: &ComputeDriverRegistry, - startup: &ServerStartupConfig, + config: &Config, + driver_startup: compute::driver_config::DriverStartupContext<'_>, ) -> Result { - resolve_configured_compute_driver( - registry, - startup.compute_driver.name(), - compute::driver_config::DriverStartupContext { - file: startup.config_file.as_ref(), - guest_tls: startup.guest_tls.as_ref(), - gateway_port: startup.config.bind_address.port(), - gateway_tls_enabled: startup.config.tls.is_some(), - endpoint_overrides: &startup.config.compute_driver_endpoints, - }, - ) + let selection = registry.select(&config.compute_drivers)?; + resolve_configured_compute_driver(registry, selection.name(), driver_startup) } fn resolve_configured_compute_driver( @@ -1681,23 +1530,6 @@ fn resolve_configured_compute_driver( Ok(ConfiguredComputeDriver::Remote { name }) } -#[cfg(any(test, feature = "in-tree-compute-drivers"))] -fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config) -> bool { - config - .gateway_jwt - .as_ref() - .is_some_and(|jwt| jwt.ttl_secs == 0) -} - -#[cfg(feature = "in-tree-compute-drivers")] -fn warn_if_kubernetes_sandbox_jwt_expiry_disabled(config: &Config) { - if kubernetes_sandbox_jwt_expiry_disabled(config) { - warn!( - "Kubernetes gateway configured with non-expiring sandbox JWTs (gateway_jwt.ttl_secs = 0); set ttl_secs > 0 for shared Kubernetes deployments" - ); - } -} - pub(crate) async fn ensure_default_workspace(store: &Store) -> Result<()> { use grpc::workspace::{DEFAULT_WORKSPACE_NAME, WORKSPACE_OBJECT_TYPE}; use openshell_core::proto::Workspace; @@ -1763,11 +1595,11 @@ mod tests { BoundGatewayListener, ConfiguredComputeDriver, ConnectionProtocol, ExtensionKind, GatewayListenerScope, MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, - is_benign_tls_handshake_failure, kubernetes_sandbox_jwt_expiry_disabled, + configured_compute_driver, is_benign_tls_handshake_failure, mint_gateway_extension_credential, serve_gateway_listener, }; use openshell_core::{ - ComputeDriverKind, Config, + Config, proto::{HealthRequest, open_shell_client::OpenShellClient}, }; use std::io::{Error, ErrorKind}; @@ -1788,6 +1620,26 @@ mod tests { tls_test_utils::{generate_test_certs_with_ca, install_rustls_provider}, }; + static DETECTION_PROBE_ORDER: LazyLock>> = + LazyLock::new(|| Mutex::new(Vec::new())); + + fn record_detection_probe(name: &'static str, available: bool) -> bool { + DETECTION_PROBE_ORDER.lock().unwrap().push(name); + available + } + + fn unavailable_first_probe() -> bool { + record_detection_probe("first", false) + } + + fn available_second_probe() -> bool { + record_detection_probe("second", true) + } + + fn available_third_probe() -> bool { + record_detection_probe("third", true) + } + fn extension_test_issuer() -> Arc { let material = openshell_bootstrap::jwt::generate_jwt_key().expect("jwt key"); Arc::new( @@ -1890,47 +1742,32 @@ mod tests { } fn test_compute_drivers() -> super::ComputeDriverRegistry { - super::install_default_compute_drivers() - } - - fn select_compute_driver( - registry: &super::ComputeDriverRegistry, - config: &Config, - driver_startup: crate::compute::driver_config::DriverStartupContext<'_>, - ) -> openshell_core::Result { - let selection = registry.select(&config.compute_drivers)?; - super::resolve_configured_compute_driver(registry, selection.name(), driver_startup) + let mut registry = super::ComputeDriverRegistry::new(); + for (name, priority) in [("alpha", 100), ("beta", 200), ("gamma", 300)] { + registry + .install( + super::ComputeDriverRegistration::new( + name, + priority, + None, + TestComputeDriverFactory, + ) + .unwrap(), + ) + .unwrap(); + } + registry } #[derive(Clone, Copy)] struct TestComputeDriverFactory; - static DETECTION_PROBE_ORDER: LazyLock>> = - LazyLock::new(|| Mutex::new(Vec::new())); - - fn record_detection_probe(name: &'static str, available: bool) -> bool { - DETECTION_PROBE_ORDER.lock().unwrap().push(name); - available - } - - fn unavailable_first_probe() -> bool { - record_detection_probe("first", false) - } - - fn available_second_probe() -> bool { - record_detection_probe("second", true) - } - - fn available_third_probe() -> bool { - record_detection_probe("third", true) - } - #[async_trait::async_trait] impl super::ComputeDriverFactory for TestComputeDriverFactory { async fn build( &self, _context: super::ComputeDriverBuildContext<'_>, - ) -> openshell_core::Result { + ) -> openshell_core::Result { unreachable!("selection tests do not construct the driver") } } @@ -2244,38 +2081,31 @@ mod tests { #[test] fn configured_compute_driver_triggers_auto_detection_when_empty() { - let config = Config::new(None).with_compute_drivers(std::iter::empty::()); - // Empty drivers triggers auto-detection, which may return Some or None - // depending on the environment. This test verifies the auto-detection path - // is taken rather than immediately returning an error. - let result = select_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ); - // Either we get a detected driver or an error about none being detected. - match result { - Ok(ConfiguredComputeDriver::Registered(registration)) => { - assert!( - matches!( - registration.name.as_str(), - "kubernetes" | "docker" | "podman" - ), - "auto-detected unexpected driver: {}", - registration.name - ); - } - Ok(ConfiguredComputeDriver::Remote { name }) => { - panic!("auto-detection returned remote driver: {name}"); - } - Err(e) => { - assert!( - e.to_string() - .contains("auto-detection found no suitable installed driver"), - "unexpected error: {e}" - ); - } + fn available() -> bool { + true } + + let mut registry = super::ComputeDriverRegistry::new(); + registry + .install( + super::ComputeDriverRegistration::new( + "detected", + 100, + Some(available), + TestComputeDriverFactory, + ) + .unwrap(), + ) + .unwrap(); + let config = Config::new(None).with_compute_drivers(std::iter::empty::()); + let result = + configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) + .unwrap(); + + let ConfiguredComputeDriver::Registered(registration) = result else { + panic!("auto-detection must select a registered driver"); + }; + assert_eq!(registration.name, "detected"); } #[test] @@ -2338,9 +2168,8 @@ mod tests { #[test] fn configured_compute_driver_rejects_multiple_entries() { - let config = Config::new(None) - .with_compute_drivers([ComputeDriverKind::Kubernetes, ComputeDriverKind::Podman]); - let err = select_compute_driver( + let config = Config::new(None).with_compute_drivers(["alpha", "beta"]); + let err = configured_compute_driver( &test_compute_drivers(), &config, test_driver_startup(&config, None), @@ -2350,13 +2179,13 @@ mod tests { err.to_string() .contains("multiple compute drivers are not supported yet") ); - assert!(err.to_string().contains("kubernetes,podman")); + assert!(err.to_string().contains("alpha,beta")); } #[test] - fn configured_compute_driver_accepts_podman() { - let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Podman]); - let driver = select_compute_driver( + fn configured_compute_driver_accepts_registered_name() { + let config = Config::new(None).with_compute_drivers(["beta"]); + let driver = configured_compute_driver( &test_compute_drivers(), &config, test_driver_startup(&config, None), @@ -2364,37 +2193,7 @@ mod tests { .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Registered(registration) if registration.name == "podman" - )); - } - - #[test] - fn configured_compute_driver_accepts_vm() { - let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Vm]); - let driver = select_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap(); - assert!(matches!( - driver, - ConfiguredComputeDriver::Registered(registration) if registration.name == "vm" - )); - } - - #[test] - fn configured_compute_driver_accepts_docker() { - let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Docker]); - let driver = select_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap(); - assert!(matches!( - driver, - ConfiguredComputeDriver::Registered(registration) if registration.name == "docker" + ConfiguredComputeDriver::Registered(registration) if registration.name == "beta" )); } @@ -2402,7 +2201,7 @@ mod tests { fn configured_compute_driver_resolves_named_remote() { let config = Config::new(None).with_compute_drivers(["kyma"]); - let driver = select_compute_driver( + let driver = configured_compute_driver( &test_compute_drivers(), &config, test_driver_startup(&config, None), @@ -2423,12 +2222,12 @@ mod tests { } #[test] - fn configured_compute_driver_uses_vm_endpoint_override() { + fn configured_compute_driver_uses_endpoint_override() { let config = Config::new(None) - .with_compute_drivers([ComputeDriverKind::Vm]) - .with_compute_driver_endpoint("vm", "/run/openshell/vm.sock"); + .with_compute_drivers(["alpha"]) + .with_compute_driver_endpoint("alpha", "/run/openshell/alpha.sock"); - let driver = select_compute_driver( + let driver = configured_compute_driver( &test_compute_drivers(), &config, test_driver_startup(&config, None), @@ -2436,17 +2235,17 @@ mod tests { .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Remote { name } if name == "vm" + ConfiguredComputeDriver::Remote { name } if name == "alpha" )); } #[test] fn configured_compute_driver_uses_builtin_endpoint_override() { let config = Config::new(None) - .with_compute_drivers([ComputeDriverKind::Docker]) - .with_compute_driver_endpoint("docker", "/run/openshell/docker.sock"); + .with_compute_drivers(["beta"]) + .with_compute_driver_endpoint("beta", "/run/openshell/beta.sock"); - let driver = select_compute_driver( + let driver = configured_compute_driver( &test_compute_drivers(), &config, test_driver_startup(&config, None), @@ -2454,48 +2253,8 @@ mod tests { .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Remote { name } if name == "docker" - )); - } - - #[test] - fn kubernetes_sandbox_jwt_expiry_disabled_warns_for_zero_ttl() { - fn config_with_jwt_ttl(ttl_secs: u64) -> Config { - let mut config = Config::new(None); - config.gateway_jwt = Some(openshell_core::GatewayJwtConfig { - signing_key_path: "/tmp/signing.pem".into(), - public_key_path: "/tmp/public.pem".into(), - kid_path: "/tmp/kid".into(), - gateway_id: "openshell".to_string(), - ttl_secs, - }); - config - } - - assert!(kubernetes_sandbox_jwt_expiry_disabled( - &config_with_jwt_ttl(0) - )); - assert!(!kubernetes_sandbox_jwt_expiry_disabled( - &config_with_jwt_ttl(3600) + ConfiguredComputeDriver::Remote { name } if name == "beta" )); - assert!(!kubernetes_sandbox_jwt_expiry_disabled(&Config::new(None))); - } - - #[cfg(target_os = "windows")] - #[test] - fn windows_builtin_compute_drivers_report_unsupported() { - for driver in [ - ComputeDriverKind::Docker, - ComputeDriverKind::Kubernetes, - ComputeDriverKind::Podman, - ComputeDriverKind::Vm, - ] { - let message = super::unsupported_builtin_compute_driver(driver).to_string(); - assert!( - message.contains("unsupported on Windows"), - "{driver} rejection should be explicit, got: {message}" - ); - } } #[tokio::test] diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index b4e8d9e506..eec3127651 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -134,11 +134,13 @@ pub fn provider_for( openshell_otel::provider_for(cfg.map(|cfg| trace_config(cfg, gateway))) } -/// Build the gateway layer while routing one selected in-process driver to -/// its own tracer provider. -pub fn layer_excluding_driver( +/// Build the `tracing` layer that forwards spans to `provider`. +/// +/// Events stay on the gateway's logging layers. Spans emitted by the +/// OpenTelemetry crates are excluded to prevent recursive export traffic. +pub fn layer( provider: &SdkTracerProvider, - driver_target_prefix: Option<&'static str>, + excluded_target_prefix: Option<&'static str>, ) -> openshell_otel::TargetOtlpLayer where S: Subscriber + for<'span> LookupSpan<'span>, @@ -146,7 +148,7 @@ where openshell_otel::layer_excluding_target_prefix( provider, INSTRUMENTATION_SCOPE, - driver_target_prefix, + excluded_target_prefix.unwrap_or("\0"), ) } @@ -180,8 +182,7 @@ pub mod test_exporter { let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() .with_simple_exporter(exporter.clone()) .build(); - let subscriber = - tracing_subscriber::registry().with(super::layer_excluding_driver(&provider, None)); + let subscriber = tracing_subscriber::registry().with(super::layer(&provider, None)); let dispatch = tracing::Dispatch::new(subscriber); TracingTestGuard { _default: tracing::dispatcher::set_default(&dispatch), diff --git a/crates/openshell-server/src/sandbox_index.rs b/crates/openshell-server/src/sandbox_index.rs index 589f88fd88..c119ca6889 100644 --- a/crates/openshell-server/src/sandbox_index.rs +++ b/crates/openshell-server/src/sandbox_index.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! In-memory indexes for correlating Kubernetes objects back to sandbox ids. +//! In-memory indexes for correlating compute resources back to sandbox ids. use std::collections::HashMap; use std::sync::{Arc, RwLock}; diff --git a/crates/openshell-server/src/tracing_setup.rs b/crates/openshell-server/src/tracing_setup.rs index edcf303072..5f9eac9a84 100644 --- a/crates/openshell-server/src/tracing_setup.rs +++ b/crates/openshell-server/src/tracing_setup.rs @@ -11,14 +11,13 @@ use opentelemetry_sdk::trace::SdkTracerProvider; use tracing_subscriber::EnvFilter; use tracing_subscriber::prelude::*; -use crate::ConfiguredComputeDriver; use crate::config_file::OtlpConfig; -use crate::otel_tracing::{GatewayResourceAttributes, SetupError}; use crate::tracing_bus::TracingLogBus; +use crate::{ComputeDriverTracingSetup, ComputeDriverTracingShutdown}; pub struct TracingHandle { tracer_provider: Option, - driver_tracer_provider: Option, + compute_driver_shutdown: Option, } impl TracingHandle { @@ -28,205 +27,45 @@ impl TracingHandle { { tracing::warn!(error = %err, "OTLP tracer provider shutdown failed"); } - if let Some(provider) = &self.driver_tracer_provider - && let Err(err) = provider.shutdown() + if let Some(shutdown) = &self.compute_driver_shutdown + && let Err(err) = shutdown() { - tracing::warn!(error = %err, "compute-driver OTLP tracer provider shutdown failed"); - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum InProcessDriverTracing { - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - Docker, - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - Kubernetes, - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - Podman, -} - -impl InProcessDriverTracing { - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - fn target_prefix(self) -> &'static str { - match self { - Self::Docker => openshell_driver_docker::otel_tracing::IN_PROCESS_TARGET_PREFIX, - Self::Kubernetes => openshell_driver_kubernetes::otel_tracing::IN_PROCESS_TARGET_PREFIX, - Self::Podman => openshell_driver_podman::otel_tracing::IN_PROCESS_TARGET_PREFIX, + tracing::warn!(error = %err, "Compute driver tracing shutdown failed"); } } } -fn in_process_driver_tracing(driver: &ConfiguredComputeDriver) -> Option { - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - match driver { - ConfiguredComputeDriver::Registered(registration) if registration.name == "docker" => { - Some(InProcessDriverTracing::Docker) - } - ConfiguredComputeDriver::Registered(registration) if registration.name == "podman" => { - Some(InProcessDriverTracing::Podman) - } - ConfiguredComputeDriver::Registered(registration) if registration.name == "kubernetes" => { - Some(InProcessDriverTracing::Kubernetes) - } - _ => None, - } - #[cfg(not(all(not(target_os = "windows"), feature = "in-tree-compute-drivers")))] - { - let _ = driver; - None - } -} - -fn in_process_driver_target_prefix(driver: Option) -> Option<&'static str> { - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - { - driver.map(InProcessDriverTracing::target_prefix) - } - #[cfg(not(all(not(target_os = "windows"), feature = "in-tree-compute-drivers")))] - { - let _ = driver; - None - } -} - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -fn in_process_driver_provider( - driver: Option, - endpoint: Option<&str>, - gateway_name: Option<&str>, -) -> (Option, Option) { - match driver { - Some(InProcessDriverTracing::Docker) => { - openshell_driver_docker::otel_tracing::provider_for(endpoint, gateway_name) - } - Some(InProcessDriverTracing::Kubernetes) => { - openshell_driver_kubernetes::otel_tracing::provider_for(endpoint, gateway_name) - } - Some(InProcessDriverTracing::Podman) => { - openshell_driver_podman::otel_tracing::provider_for(endpoint, gateway_name) - } - None => (None, None), - } -} - -#[cfg(not(all(not(target_os = "windows"), feature = "in-tree-compute-drivers")))] -fn in_process_driver_provider( - _driver: Option, - _endpoint: Option<&str>, - _gateway_name: Option<&str>, -) -> (Option, Option) { - (None, None) -} - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -fn in_process_driver_layer( - provider: &Option, - driver: Option, -) -> Option> -where - S: tracing::Subscriber + for<'span> tracing_subscriber::registry::LookupSpan<'span>, -{ - provider.as_ref().map(|provider| match driver { - Some(InProcessDriverTracing::Docker) => { - openshell_driver_docker::otel_tracing::in_process_layer(provider) - } - Some(InProcessDriverTracing::Kubernetes) => { - openshell_driver_kubernetes::otel_tracing::in_process_layer(provider) - } - Some(InProcessDriverTracing::Podman) => { - openshell_driver_podman::otel_tracing::in_process_layer(provider) - } - None => unreachable!("a driver provider requires a selected driver"), - }) -} - -#[cfg(not(all(not(target_os = "windows"), feature = "in-tree-compute-drivers")))] -fn in_process_driver_layer( - _provider: &Option, - _driver: Option, -) -> Option> -where - S: tracing::Subscriber + for<'span> tracing_subscriber::registry::LookupSpan<'span>, -{ - None -} - pub fn install( env_filter: EnvFilter, tracing_log_bus: &TracingLogBus, otlp_config: Option<&OtlpConfig>, - driver: &ConfiguredComputeDriver, - gateway: GatewayResourceAttributes<'_>, -) -> (TracingHandle, Option) { - let (tracer_provider, setup_error) = crate::otel_tracing::provider_for(otlp_config, gateway); - let selected_driver = in_process_driver_tracing(driver); - let driver_endpoint = selected_driver - .is_some() - .then_some(otlp_config) - .flatten() - .map(|config| config.endpoint.as_str()); - let (driver_tracer_provider, driver_setup_error) = - in_process_driver_provider(selected_driver, driver_endpoint, gateway.name()); + compute_driver_tracing: ComputeDriverTracingSetup, +) -> (TracingHandle, Option) { + let (tracer_provider, setup_error) = crate::otel_tracing::provider_for(otlp_config); + let ComputeDriverTracingSetup { + layer, + shutdown, + error, + target_prefix, + } = compute_driver_tracing; tracing_subscriber::registry() + .with(layer) .with(env_filter) .with(tracing_subscriber::fmt::layer()) .with(tracing_log_bus.layer()) - .with(tracer_provider.as_ref().map(|provider| { - crate::otel_tracing::layer_excluding_driver( - provider, - in_process_driver_target_prefix(selected_driver), - ) - })) - .with(in_process_driver_layer( - &driver_tracer_provider, - selected_driver, - )) + .with( + tracer_provider + .as_ref() + .map(|provider| crate::otel_tracing::layer(provider, target_prefix)), + ) .init(); ( TracingHandle { tracer_provider, - driver_tracer_provider, + compute_driver_shutdown: shutdown, }, - setup_error.or(driver_setup_error), + setup_error.map(|error| error.to_string()).or(error), ) } - -#[cfg(test)] -mod tests { - use super::*; - - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - #[test] - fn in_process_driver_tracing_selects_registered_compute_drivers() { - let registry = crate::install_default_compute_drivers(); - let registered = |name| { - ConfiguredComputeDriver::Registered( - registry - .get(name) - .unwrap_or_else(|| panic!("{name} driver is registered")) - .clone(), - ) - }; - assert_eq!( - in_process_driver_tracing(®istered("podman")), - Some(InProcessDriverTracing::Podman) - ); - assert_eq!( - in_process_driver_tracing(®istered("docker")), - Some(InProcessDriverTracing::Docker) - ); - assert_eq!( - in_process_driver_tracing(®istered("kubernetes")), - Some(InProcessDriverTracing::Kubernetes) - ); - assert_eq!( - in_process_driver_tracing(&ConfiguredComputeDriver::Remote { - name: "custom".to_string(), - }), - None - ); - } -} diff --git a/deploy/docker/Dockerfile.gateway-macos b/deploy/docker/Dockerfile.gateway-macos index 122f16eba8..c7d526a039 100644 --- a/deploy/docker/Dockerfile.gateway-macos +++ b/deploy/docker/Dockerfile.gateway-macos @@ -53,6 +53,7 @@ ENV BINDGEN_EXTRA_CLANG_ARGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ - COPY Cargo.toml Cargo.lock ./ COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml +COPY crates/openshell-gateway/Cargo.toml crates/openshell-gateway/Cargo.toml COPY crates/openshell-driver-kubernetes/Cargo.toml crates/openshell-driver-kubernetes/Cargo.toml COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml COPY crates/openshell-prover/Cargo.toml crates/openshell-prover/Cargo.toml @@ -61,39 +62,42 @@ COPY crates/openshell-server/Cargo.toml crates/openshell-server/Cargo.toml COPY crates/openshell-core/build.rs crates/openshell-core/build.rs COPY proto/ proto/ -RUN sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-server", "crates/openshell-core", "crates/openshell-driver-kubernetes", "crates/openshell-policy", "crates/openshell-prover", "crates/openshell-router"]|' Cargo.toml +RUN sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-gateway", "crates/openshell-server", "crates/openshell-core", "crates/openshell-driver-kubernetes", "crates/openshell-policy", "crates/openshell-prover", "crates/openshell-router"]|' Cargo.toml RUN mkdir -p crates/openshell-core/src \ + crates/openshell-gateway/src \ crates/openshell-driver-kubernetes/src \ crates/openshell-policy/src \ crates/openshell-prover/src \ crates/openshell-router/src \ crates/openshell-server/src && \ touch crates/openshell-core/src/lib.rs && \ + touch crates/openshell-gateway/src/lib.rs && \ + printf 'fn main() {}\n' > crates/openshell-gateway/src/main.rs && \ touch crates/openshell-driver-kubernetes/src/lib.rs && \ printf 'fn main() {}\n' > crates/openshell-driver-kubernetes/src/main.rs && \ touch crates/openshell-policy/src/lib.rs && \ touch crates/openshell-prover/src/lib.rs && \ touch crates/openshell-router/src/lib.rs && \ - touch crates/openshell-server/src/lib.rs && \ - printf 'fn main() {}\n' > crates/openshell-server/src/main.rs + touch crates/openshell-server/src/lib.rs RUN --mount=type=cache,id=cargo-registry-gateway-macos,sharing=locked,target=/root/.cargo/registry \ --mount=type=cache,id=cargo-git-gateway-macos,sharing=locked,target=/root/.cargo/git \ --mount=type=cache,id=cargo-target-gateway-macos-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - cargo build --release --target aarch64-apple-darwin -p openshell-server --features bundled-z3 2>/dev/null || true + cargo build --release --target aarch64-apple-darwin -p openshell-gateway --features bundled-z3 2>/dev/null || true COPY crates/ crates/ COPY providers/ providers/ RUN touch crates/openshell-core/src/lib.rs \ + crates/openshell-gateway/src/lib.rs \ + crates/openshell-gateway/src/main.rs \ crates/openshell-driver-kubernetes/src/lib.rs \ crates/openshell-driver-kubernetes/src/main.rs \ crates/openshell-policy/src/lib.rs \ crates/openshell-prover/src/lib.rs \ crates/openshell-router/src/lib.rs \ crates/openshell-server/src/lib.rs \ - crates/openshell-server/src/main.rs \ crates/openshell-core/build.rs \ proto/*.proto @@ -105,7 +109,7 @@ RUN --mount=type=cache,id=cargo-registry-gateway-macos,sharing=locked,target=/ro if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ fi && \ - cargo build --release --target aarch64-apple-darwin -p openshell-server --features bundled-z3 && \ + cargo build --release --target aarch64-apple-darwin -p openshell-gateway --features bundled-z3 && \ cp target/aarch64-apple-darwin/release/openshell-gateway /openshell-gateway FROM scratch AS binary diff --git a/e2e/no-compute-driver-gateway.sh b/e2e/no-compute-driver-gateway.sh index 7ad3896ca1..baa4a7d193 100755 --- a/e2e/no-compute-driver-gateway.sh +++ b/e2e/no-compute-driver-gateway.sh @@ -8,11 +8,13 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${ROOT}" echo "Building gateway without compiled compute drivers..." -cargo build -p openshell-server --bin openshell-gateway \ +cargo build -p openshell-gateway --bin openshell-gateway \ --no-default-features --features telemetry +cargo check -p openshell-core --no-default-features --all-targets -dependency_tree="$(cargo tree -p openshell-server \ +dependency_tree="$(cargo tree -p openshell-gateway \ --no-default-features --features telemetry --edges normal)" +server_dependency_tree="$(cargo tree -p openshell-server --edges normal)" for driver in \ openshell-driver-docker \ openshell-driver-kubernetes \ @@ -22,7 +24,18 @@ for driver in \ echo "ERROR: driver-free gateway dependency graph contains ${driver}" >&2 exit 1 fi + if grep -q "${driver} v" <<<"${server_dependency_tree}"; then + echo "ERROR: openshell-server dependency graph contains ${driver}" >&2 + exit 1 + fi done +if rg -n \ + 'ComputeDriverKind|openshell_driver_(docker|podman|kubernetes)([^_[:alnum:]]|$)|ComputeRuntime::new_(docker|podman|kubernetes)|VmComputeConfig|compute::vm|driver_config::builtin|libkrun|gvproxy|qemu' \ + crates/openshell-core crates/openshell-server; then + echo "ERROR: backend-specific compute-driver knowledge leaked into core/server" >&2 + exit 1 +fi + "${ROOT}/target/debug/openshell-gateway" --version echo "Driver-free gateway build passed." diff --git a/e2e/run.sh b/e2e/run.sh index 0505730f05..875186394c 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -250,7 +250,7 @@ guest_gateway_bin= if [ "${mode}" = host ]; then echo "==> Building native host openshell-gateway" mise x -- cargo build "${cargo_jobs[@]}" \ - -p openshell-server \ + -p openshell-gateway \ --bin openshell-gateway \ --features bundled-z3 host_gateway_bin="${target_dir}/debug/openshell-gateway" @@ -268,7 +268,7 @@ else mise x -- cargo zigbuild "${cargo_jobs[@]}" \ --release \ --target "${linux_gateway_zig_target}" \ - -p openshell-server \ + -p openshell-gateway \ --bin openshell-gateway \ --features bundled-z3 ) diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index de7e3ad09d..9acc633d65 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -103,10 +103,10 @@ if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then echo "==> Building driver-free openshell-gateway" cargo build \ - -p openshell-server --bin openshell-gateway \ - --no-default-features + -p openshell-gateway --bin openshell-gateway \ + --no-default-features --features telemetry else - build_packages+=(-p openshell-server) + build_packages+=(-p openshell-gateway) fi else echo "==> Using prebuilt openshell-gateway at ${GATEWAY_BIN}" diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index c512dab60a..b1f62e380b 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -219,11 +219,11 @@ e2e_build_gateway_binaries() { echo "Building openshell-gateway..." if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then cargo build "${jobs[@]}" \ - -p openshell-server --bin openshell-gateway \ - --no-default-features + -p openshell-gateway --bin openshell-gateway \ + --no-default-features --features telemetry else cargo build "${jobs[@]}" \ - -p openshell-server --bin openshell-gateway + -p openshell-gateway --bin openshell-gateway fi else echo "Using prebuilt openshell gateway at ${OPENSHELL_GATEWAY_BIN}" diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 08dd08113e..bd7f246d24 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -642,9 +642,12 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then fi external_gateway="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}" external_driver="${OPENSHELL_EXTERNAL_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-kubernetes}" + # The test image uses a distroless runtime, so keep Z3 self-contained just + # like the production gateway image artifact. A host-linked debug binary + # would otherwise require libz3.so from the CI build machine at runtime. if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then - cargo build -p openshell-server --bin openshell-gateway \ - --no-default-features --features bundled-z3 + cargo build -p openshell-gateway --bin openshell-gateway \ + --no-default-features --features telemetry,bundled-z3 fi if [ -z "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" ]; then cargo build -p openshell-driver-kubernetes --bin openshell-driver-kubernetes diff --git a/examples/governance-interceptor/smoke.sh b/examples/governance-interceptor/smoke.sh index feaa42df05..e25ee0d3cb 100755 --- a/examples/governance-interceptor/smoke.sh +++ b/examples/governance-interceptor/smoke.sh @@ -649,7 +649,7 @@ wait_until_stopped() { cd "$ROOT" -run_setup_step "building gateway" cargo build --quiet -p openshell-server --bin openshell-gateway +run_setup_step "building gateway" cargo build --quiet -p openshell-gateway --bin openshell-gateway run_setup_step "building governance interceptor" cargo build --quiet --manifest-path "$EXAMPLE_DIR/Cargo.toml" run_setup_step "building CLI" cargo build --quiet -p openshell-cli --bin openshell diff --git a/examples/supervisor-middleware-content-guard/smoke.sh b/examples/supervisor-middleware-content-guard/smoke.sh index 509ffd403d..b30475c8ca 100755 --- a/examples/supervisor-middleware-content-guard/smoke.sh +++ b/examples/supervisor-middleware-content-guard/smoke.sh @@ -444,7 +444,7 @@ EXAMPLE_TARGET_DIR="$(cargo_target_dir "$EXAMPLE_DIR/Cargo.toml")" GATEWAY_BIN="$ROOT_TARGET_DIR/debug/openshell-gateway" CLI_BIN="$ROOT_TARGET_DIR/debug/openshell" MIDDLEWARE_BIN="$EXAMPLE_TARGET_DIR/debug/supervisor-middleware-content-guard" -run_setup_step "building gateway" cargo build --quiet -p openshell-server --bin openshell-gateway +run_setup_step "building gateway" cargo build --quiet -p openshell-gateway --bin openshell-gateway run_setup_step "building content guard" cargo build --quiet --manifest-path "$EXAMPLE_DIR/Cargo.toml" run_setup_step "building CLI" cargo build --quiet -p openshell-cli --bin openshell generate_gateway_jwt_bundle diff --git a/tasks/ci.toml b/tasks/ci.toml index 4c0b5f8ea7..cc4158448f 100644 --- a/tasks/ci.toml +++ b/tasks/ci.toml @@ -31,7 +31,7 @@ hide = true description = "Build release Rust binaries consumed by the hand-staged snap" run = [ "cargo build --release -p openshell-cli", - "cargo build --release -p openshell-server --features bundled-z3", + "cargo build --release -p openshell-gateway --features bundled-z3", "cargo build --release -p openshell-sandbox", ] diff --git a/tasks/gateway.toml b/tasks/gateway.toml index fb88849348..bbf309c459 100644 --- a/tasks/gateway.toml +++ b/tasks/gateway.toml @@ -5,7 +5,7 @@ ["build:gateway"] description = "Build the standalone openshell-gateway binary" -run = "cargo build -p openshell-server --bin openshell-gateway" +run = "cargo build -p openshell-gateway --bin openshell-gateway" hide = true ["gateway"] diff --git a/tasks/rust.toml b/tasks/rust.toml index 6ef34f60df..03b7ebb7ce 100644 --- a/tasks/rust.toml +++ b/tasks/rust.toml @@ -51,12 +51,12 @@ description = "Verify telemetry emission code is compiled out with --no-default- run = [ # Positive control: the default (telemetry-on) gateway must contain the # markers, so the absent checks below can never become silently vacuous. - "cargo build -p openshell-server --bin openshell-gateway", + "cargo build -p openshell-gateway --bin openshell-gateway", "tasks/scripts/verify-telemetry-compiled-out.sh present target/debug/openshell-gateway", # Guard: telemetry-free builds must contain no telemetry markers. Built # through the `defaults-without-telemetry` alias, which is how the docs tell # operators to produce these artifacts. - "cargo build -p openshell-server --bin openshell-gateway --no-default-features --features defaults-without-telemetry", + "cargo build -p openshell-gateway --bin openshell-gateway --no-default-features --features defaults-without-telemetry", "tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-gateway", "cargo build -p openshell-sandbox --bin openshell-sandbox --no-default-features --features defaults-without-telemetry", "tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-sandbox", diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index b38bfb7942..6826829fd8 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -161,7 +161,7 @@ fi echo "Building openshell-gateway..." cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \ - -p openshell-server --bin openshell-gateway + -p openshell-gateway --bin openshell-gateway TLS_DIR="${STATE_DIR}/tls" echo "Generating local gateway credentials..." diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 2e8aecc5d0..3818dca364 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -311,7 +311,7 @@ fi echo "==> Building openshell-gateway and openshell-driver-vm" cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \ - -p openshell-server -p openshell-driver-vm + -p openshell-gateway -p openshell-driver-vm if [ "$(uname -s)" = "Darwin" ]; then echo "==> Codesigning openshell-driver-vm (Hypervisor entitlement)" diff --git a/tasks/scripts/package-deb-install.sh b/tasks/scripts/package-deb-install.sh index b6e4730674..e20d409bbd 100755 --- a/tasks/scripts/package-deb-install.sh +++ b/tasks/scripts/package-deb-install.sh @@ -56,7 +56,7 @@ remove_existing_gateway_registration() { echo "==> Building release binaries" cargo build --release \ -p openshell-cli \ - -p openshell-server \ + -p openshell-gateway \ -p openshell-driver-vm echo "==> Building Debian package" diff --git a/tasks/scripts/stage-prebuilt-binaries.sh b/tasks/scripts/stage-prebuilt-binaries.sh index 757a21298b..fe4913439a 100755 --- a/tasks/scripts/stage-prebuilt-binaries.sh +++ b/tasks/scripts/stage-prebuilt-binaries.sh @@ -124,7 +124,7 @@ components_for_target() { resolve_component() { case "$1" in gateway) - crate=openshell-server + crate=openshell-gateway binary=openshell-gateway target_libc=gnu ;; diff --git a/tasks/scripts/verify-defaults-without-telemetry.sh b/tasks/scripts/verify-defaults-without-telemetry.sh index 4d78cbc5b4..1fd7e67dff 100755 --- a/tasks/scripts/verify-defaults-without-telemetry.sh +++ b/tasks/scripts/verify-defaults-without-telemetry.sh @@ -22,7 +22,7 @@ set -euo pipefail # Crates that define the alias. Each must forward `telemetry` and define # `defaults-without-telemetry`. CRATES=( - openshell-server + openshell-gateway openshell-sandbox openshell-driver-vm ) diff --git a/tasks/scripts/vm/smoke-orphan-cleanup.sh b/tasks/scripts/vm/smoke-orphan-cleanup.sh index 6da48919d1..7d0b05334d 100755 --- a/tasks/scripts/vm/smoke-orphan-cleanup.sh +++ b/tasks/scripts/vm/smoke-orphan-cleanup.sh @@ -37,7 +37,7 @@ trap cleanup_stray EXIT build_binaries() { echo "==> Ensuring binaries are built" if [ ! -x "$ROOT/target/debug/openshell-gateway" ] || [ ! -x "$ROOT/target/debug/openshell-driver-vm" ]; then - cargo build -p openshell-server -p openshell-driver-vm >&2 + cargo build -p openshell-gateway -p openshell-driver-vm >&2 fi if [ "$(uname -s)" = "Darwin" ]; then codesign \ From eeb0584d685129433af92741e78d71a5f5d5246e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 19 Aug 2026 22:08:02 -0700 Subject: [PATCH 2/9] fix(telemetry): bound compute driver categories Signed-off-by: Drew Newberry --- architecture/build.md | 2 +- crates/openshell-core/src/telemetry.rs | 51 +++++++------ crates/openshell-gateway/BUILD.bazel | 64 ---------------- crates/openshell-gateway/src/lib.rs | 17 +++-- .../src/auth/compute_driver.rs | 2 +- crates/openshell-server/src/compute/mod.rs | 24 ++++-- crates/openshell-server/src/grpc/sandbox.rs | 33 +------- crates/openshell-server/src/lib.rs | 76 ++++++++++++++----- 8 files changed, 115 insertions(+), 154 deletions(-) delete mode 100644 crates/openshell-gateway/BUILD.bazel diff --git a/architecture/build.md b/architecture/build.md index 355892abf9..393eb9e468 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -10,7 +10,7 @@ OpenShell builds these main artifacts: | Artifact | Source | |---|---| -| Gateway binary | `crates/openshell-server` | +| Gateway binary | `crates/openshell-gateway` | | CLI binaries and system packages | `crates/openshell-cli` plus release packaging | | E2E conformance CLI | `crates/openshell-conformance-cli` | | Python SDK wheel | `python/openshell` | diff --git a/crates/openshell-core/src/telemetry.rs b/crates/openshell-core/src/telemetry.rs index f092f5f8aa..780e5c7920 100644 --- a/crates/openshell-core/src/telemetry.rs +++ b/crates/openshell-core/src/telemetry.rs @@ -159,20 +159,35 @@ impl SandboxTemplateSource { } } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TelemetryComputeDriver(String); +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TelemetryComputeDriver(&'static str); impl TelemetryComputeDriver { #[must_use] - pub fn as_str(&self) -> &str { - &self.0 + pub const fn as_str(self) -> &'static str { + self.0 } + /// Classify an unregistered compute driver without exposing its configured + /// name. #[must_use] - pub fn from_raw(raw: &str) -> Self { - let name = crate::config::normalize_compute_driver_name(raw) - .unwrap_or_else(|_| "unknown".to_string()); - Self(name) + pub const fn custom() -> Self { + Self("custom") + } + + /// Define a bounded, anonymous category at a binary composition boundary. + /// + /// The category must be a static operational label. Never construct it + /// from user input, configuration, resource names, or other runtime data. + #[must_use] + pub const fn anonymous_category(category: &'static str) -> Self { + Self(category) + } +} + +impl Default for TelemetryComputeDriver { + fn default() -> Self { + Self::custom() } } @@ -657,21 +672,11 @@ mod tests { } #[test] - fn compute_driver_values_are_normalized_without_enumerating_backends() { - assert_eq!(TelemetryComputeDriver::from_raw("alpha").as_str(), "alpha"); - assert_eq!( - TelemetryComputeDriver::from_raw(" Alpha ").as_str(), - "alpha" - ); - assert_eq!( - TelemetryComputeDriver::from_raw("CUSTOM_BACKEND").as_str(), - "custom_backend" - ); - assert_eq!(TelemetryComputeDriver::from_raw("beta").as_str(), "beta"); - assert_eq!(TelemetryComputeDriver::from_raw("gamma").as_str(), "gamma"); + fn compute_driver_values_are_bounded_by_the_composition_boundary() { + assert_eq!(TelemetryComputeDriver::custom().as_str(), "custom"); assert_eq!( - TelemetryComputeDriver::from_raw("private-driver").as_str(), - "private-driver" + TelemetryComputeDriver::anonymous_category("first_party").as_str(), + "first_party" ); } @@ -760,7 +765,7 @@ mod disabled_tests { 1, false, SandboxTemplateSource::Default, - TelemetryComputeDriver::from_raw("test-driver"), + TelemetryComputeDriver::custom(), ); emit_policy_decision( PolicyDecisionOperation::Approve, diff --git a/crates/openshell-gateway/BUILD.bazel b/crates/openshell-gateway/BUILD.bazel deleted file mode 100644 index 90b464ef15..0000000000 --- a/crates/openshell-gateway/BUILD.bazel +++ /dev/null @@ -1,64 +0,0 @@ -load("@crates//:defs.bzl", "aliases", "all_crate_deps") -load("@rules_rs//rs:rust_binary.bzl", "rust_binary") -load("@rules_rs//rs:rust_library.bzl", "rust_library") -load("@rules_rs//rs:rust_test.bzl", "rust_test") -load("@rules_rust//rust:defs.bzl", "rustfmt_test") -load("@workspace_version//:version.bzl", "WORKSPACE_VERSION") - -rust_library( - name = "openshell-gateway", - srcs = glob( - ["src/**/*.rs"], - exclude = ["src/main.rs"], - ), - aliases = aliases(), - crate_features = [ - "in-tree-compute-drivers", - "telemetry", - ], - version = WORKSPACE_VERSION, - visibility = ["//visibility:public"], - deps = all_crate_deps(normal = True), -) - -rust_binary( - name = "openshell-gateway-bin", - srcs = ["src/main.rs"], - aliases = aliases(), - binary_name = "openshell-gateway", - version = WORKSPACE_VERSION, - visibility = ["//visibility:public"], - deps = all_crate_deps(normal = True) + [":openshell-gateway"], -) - -rust_test( - name = "openshell-gateway_lib_test", - crate = ":openshell-gateway", - crate_features = [ - "in-tree-compute-drivers", - "telemetry", - ], - deps = all_crate_deps(normal_dev = True), -) - -rust_test( - name = "openshell-gateway_bin_test", - srcs = ["src/main.rs"], - aliases = aliases(), - version = WORKSPACE_VERSION, - deps = all_crate_deps( - normal = True, - normal_dev = True, - ) + [":openshell-gateway"], -) - -rustfmt_test( - name = "rustfmt_test", - targets = [ - ":openshell-gateway", - ":openshell-gateway-bin", - ":openshell-gateway_bin_test", - ":openshell-gateway_lib_test", - ], - visibility = ["//crates:__pkg__"], -) diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index c292c208a4..a573a18aed 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -19,6 +19,8 @@ compile_error!( #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] mod vm; +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +use openshell_core::telemetry::TelemetryComputeDriver; #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] use openshell_server::ComputeDriverRegistration; use openshell_server::ComputeDriverRegistry; @@ -44,6 +46,7 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { ) .map(|registration| { registration + .with_telemetry_category(TelemetryComputeDriver::anonymous_category("kubernetes")) .without_mtls_user_auth() .with_inherited_config_keys(&[ "namespace", @@ -64,6 +67,7 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { ) .map(|registration| { registration + .with_telemetry_category(TelemetryComputeDriver::anonymous_category("podman")) .with_local_singleplayer() .with_tracing_setup(podman_tracing_setup) .with_inherited_config_keys(&[ @@ -83,6 +87,7 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { ) .map(|registration| { registration + .with_telemetry_category(TelemetryComputeDriver::anonymous_category("docker")) .with_local_singleplayer() .with_inherited_config_keys(&[ "sandbox_namespace", @@ -96,6 +101,7 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { }), ComputeDriverRegistration::new("vm", u16::MAX, None, VmFactory).map(|registration| { registration + .with_telemetry_category(TelemetryComputeDriver::anonymous_category("vm")) .with_local_singleplayer() .with_inherited_config_keys(&[ "default_image", @@ -118,18 +124,13 @@ fn podman_tracing_setup( let (provider, error) = openshell_driver_podman::otel_tracing::provider_for(otlp_endpoint); let layer = provider.as_ref().map(|provider| { let layer: openshell_server::ComputeDriverTracingLayer = Box::new( - openshell_driver_podman::otel_tracing::in_process_layer( - provider, - ), + openshell_driver_podman::otel_tracing::in_process_layer(provider), ); layer }); let shutdown = provider.map(|provider| { - let shutdown: openshell_server::ComputeDriverTracingShutdown = Box::new(move || { - provider - .shutdown() - .map_err(|error| error.to_string()) - }); + let shutdown: openshell_server::ComputeDriverTracingShutdown = + Box::new(move || provider.shutdown().map_err(|error| error.to_string())); shutdown }); openshell_server::ComputeDriverTracingSetup::new( diff --git a/crates/openshell-server/src/auth/compute_driver.rs b/crates/openshell-server/src/auth/compute_driver.rs index cedc2115b5..04caee61b1 100644 --- a/crates/openshell-server/src/auth/compute_driver.rs +++ b/crates/openshell-server/src/auth/compute_driver.rs @@ -52,7 +52,7 @@ impl Authenticator for ComputeDriverAuthenticator { Ok(Some(Principal::Sandbox(SandboxPrincipal { sandbox_id, source: SandboxIdentitySource::ComputeDriver { - driver_name: self.compute.selected_driver_name().to_string(), + driver_name: self.compute.configured_driver_name().to_string(), }, trust_domain: Some("openshell".to_string()), }))) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 8fd15d944c..aca1b04f72 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -39,6 +39,7 @@ use openshell_core::proto::{ PlatformEvent, Sandbox, SandboxCondition, SandboxPhase, SandboxSpec, SandboxStatus, SandboxTemplate, ServiceEndpoint, SshSession, }; +use openshell_core::telemetry::TelemetryComputeDriver; use openshell_core::{ObjectLabels, ObjectWorkspace}; use prost::Message; use std::collections::HashMap; @@ -551,6 +552,7 @@ impl ComputeDriver for RemoteComputeDriver { pub struct ComputeRuntime { driver: TracedDriver, driver_info: ComputeDriverInfoSnapshot, + telemetry_compute_driver: TelemetryComputeDriver, driver_process: Option>, default_image: String, store: Arc, @@ -676,6 +678,7 @@ impl ComputeRuntime { Ok(Self { driver: TracedDriver::new(driver, driver_name), driver_info, + telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process, default_image, store, @@ -754,11 +757,6 @@ impl ComputeRuntime { &self.driver_info.name } - #[must_use] - pub fn selected_driver_name(&self) -> &str { - &self.driver_info.name - } - #[must_use] pub fn supports_sandbox_authentication(&self) -> bool { self.driver_info.supports_sandbox_authentication @@ -781,6 +779,20 @@ impl ComputeRuntime { .map(|response| response.into_inner().sandbox_id) } + #[must_use] + pub(crate) fn telemetry_compute_driver(&self) -> TelemetryComputeDriver { + self.telemetry_compute_driver + } + + #[must_use] + pub(crate) fn with_telemetry_compute_driver( + mut self, + telemetry_compute_driver: TelemetryComputeDriver, + ) -> Self { + self.telemetry_compute_driver = telemetry_compute_driver; + self + } + #[must_use] pub(crate) fn gateway_listener_requirements(&self) -> &[GatewayListenerRequirement] { &self.gateway_listener_requirements @@ -4628,6 +4640,7 @@ pub async fn new_test_runtime_with_driver( gateway_manages_lifecycle: false, supports_sandbox_authentication, }, + telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process: None, default_image: "openshell/sandbox:test".to_string(), store, @@ -5355,6 +5368,7 @@ mod tests { gateway_manages_lifecycle: false, supports_sandbox_authentication: false, }, + telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process: None, default_image: "openshell/sandbox:test".to_string(), store, diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 227db094a0..5364e0541e 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -29,8 +29,7 @@ use openshell_core::proto::{ }; use openshell_core::proto::{Sandbox, SandboxPhase, SandboxTemplate, SshSession}; use openshell_core::telemetry::{ - LifecycleOperation, LifecycleResource, SandboxTemplateSource, TelemetryComputeDriver, - TelemetryOutcome, + LifecycleOperation, LifecycleResource, SandboxTemplateSource, TelemetryOutcome, }; use openshell_core::{ObjectId, ObjectName, ObjectWorkspace}; use prost::Message; @@ -172,7 +171,7 @@ fn emit_sandbox_create_telemetry( request: &CreateSandboxRequest, outcome: TelemetryOutcome, ) { - let compute_driver = telemetry_compute_driver(state.compute.configured_driver_name()); + let compute_driver = state.compute.telemetry_compute_driver(); let Some(spec) = request.spec.as_ref() else { openshell_core::telemetry::emit_sandbox_create( outcome, @@ -205,10 +204,6 @@ fn emit_sandbox_create_telemetry( ); } -fn telemetry_compute_driver(driver_name: &str) -> TelemetryComputeDriver { - TelemetryComputeDriver::from_raw(driver_name) -} - async fn handle_create_sandbox_inner( state: &Arc, request: Request, @@ -2595,30 +2590,6 @@ mod tests { } } - #[test] - fn telemetry_compute_driver_uses_resolved_driver_kind() { - assert_eq!( - telemetry_compute_driver("docker"), - TelemetryComputeDriver::from_raw("docker") - ); - assert_eq!( - telemetry_compute_driver("kubernetes"), - TelemetryComputeDriver::from_raw("kubernetes") - ); - assert_eq!( - telemetry_compute_driver("podman"), - TelemetryComputeDriver::from_raw("podman") - ); - assert_eq!( - telemetry_compute_driver("vm"), - TelemetryComputeDriver::from_raw("vm") - ); - assert_eq!( - telemetry_compute_driver(""), - TelemetryComputeDriver::from_raw("") - ); - } - #[test] fn shell_escape_safe_chars_pass_through() { assert_eq!(shell_escape("ls").unwrap(), "ls"); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 3c1c5c64dc..cf1ad92bfa 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -49,6 +49,7 @@ mod ws_tunnel; use metrics_exporter_prometheus::PrometheusBuilder; use openshell_core::net::set_tcp_nodelay_best_effort; +use openshell_core::telemetry::TelemetryComputeDriver; use openshell_core::{Config, Error, ObjectLabels, Result}; use openshell_extension_core::{ BearerTokenSlot, ExtensionAudience, ExtensionCallerKind, ExtensionKind, MAX_EXTENSION_TOKEN_TTL, @@ -671,7 +672,7 @@ pub(crate) async fn run_server( auth::compute_driver::ComputeDriverAuthenticator::new(state.compute.clone()), )); info!( - driver = state.compute.selected_driver_name(), + driver = state.compute.configured_driver_name(), "compute-driver sandbox bootstrap authenticator enabled" ); } @@ -1104,6 +1105,7 @@ pub struct ComputeDriverRegistration { detection_priority: u16, detect: Option bool>, factory: Arc, + telemetry_category: TelemetryComputeDriver, inherited_config_keys: &'static [&'static str], local_singleplayer: bool, supports_mtls_user_auth: bool, @@ -1136,6 +1138,7 @@ impl ComputeDriverRegistration { detection_priority, detect, factory: Arc::new(factory), + telemetry_category: TelemetryComputeDriver::custom(), inherited_config_keys: &[], local_singleplayer: false, supports_mtls_user_auth: true, @@ -1150,6 +1153,14 @@ impl ComputeDriverRegistration { self } + /// Assign a bounded telemetry category chosen by the binary composition + /// boundary. Runtime driver names are never used as telemetry values. + #[must_use] + pub fn with_telemetry_category(mut self, category: TelemetryComputeDriver) -> Self { + self.telemetry_category = category; + self + } + /// Mark a backend whose local deployment should use single-player defaults. #[must_use] pub fn with_local_singleplayer(mut self) -> Self { @@ -1391,6 +1402,7 @@ async fn build_compute_runtime( shutdown_rx: watch::Receiver, ) -> Result { let driver = resolve_configured_compute_driver(registry, selection.name(), driver_startup)?; + let telemetry_compute_driver = driver.telemetry_compute_driver(registry); info!(driver = %driver.name(), "Using compute driver"); if config .gateway_jwt @@ -1472,7 +1484,7 @@ async fn build_compute_runtime( } }; - Ok(runtime) + Ok(runtime.with_telemetry_compute_driver(telemetry_compute_driver)) } #[derive(Debug, Clone)] @@ -1497,6 +1509,17 @@ impl ConfiguredComputeDriver { .is_some_and(ComputeDriverRegistration::is_local_singleplayer), } } + + fn telemetry_compute_driver(&self, registry: &ComputeDriverRegistry) -> TelemetryComputeDriver { + match self { + Self::Registered(registration) => registration.telemetry_category, + Self::Remote { name } => registry + .get(name) + .map_or_else(TelemetryComputeDriver::custom, |registration| { + registration.telemetry_category + }), + } + } } #[cfg(test)] @@ -1752,7 +1775,12 @@ mod tests { None, TestComputeDriverFactory, ) - .unwrap(), + .unwrap() + .with_telemetry_category( + openshell_core::telemetry::TelemetryComputeDriver::anonymous_category( + "registered", + ), + ), ) .unwrap(); } @@ -2185,12 +2213,14 @@ mod tests { #[test] fn configured_compute_driver_accepts_registered_name() { let config = Config::new(None).with_compute_drivers(["beta"]); - let driver = configured_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap(); + let registry = test_compute_drivers(); + let driver = + configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) + .unwrap(); + assert_eq!( + driver.telemetry_compute_driver(®istry).as_str(), + "registered" + ); assert!(matches!( driver, ConfiguredComputeDriver::Registered(registration) if registration.name == "beta" @@ -2200,13 +2230,15 @@ mod tests { #[test] fn configured_compute_driver_resolves_named_remote() { let config = Config::new(None).with_compute_drivers(["kyma"]); + let registry = test_compute_drivers(); - let driver = configured_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap(); + let driver = + configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) + .unwrap(); + assert_eq!( + driver.telemetry_compute_driver(®istry).as_str(), + "custom" + ); match driver { ConfiguredComputeDriver::Remote { name } => { @@ -2226,13 +2258,15 @@ mod tests { let config = Config::new(None) .with_compute_drivers(["alpha"]) .with_compute_driver_endpoint("alpha", "/run/openshell/alpha.sock"); + let registry = test_compute_drivers(); - let driver = configured_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap(); + let driver = + configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) + .unwrap(); + assert_eq!( + driver.telemetry_compute_driver(®istry).as_str(), + "registered" + ); assert!(matches!( driver, ConfiguredComputeDriver::Remote { name } if name == "alpha" From 10d8235e0ec1abc01f6c9b196dc3bc5c35749a7c Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 20 Aug 2026 17:56:08 -0700 Subject: [PATCH 3/9] refactor(core): keep runtime transport generic Signed-off-by: Drew Newberry --- crates/openshell-core/src/sandbox_env.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 8da9a7d2a8..99ac55fe6e 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -105,7 +105,7 @@ impl MainProcessConfig { } /// Encode the versioned transport without whitespace for constrained - /// environment-variable transports such as libkrun. + /// environment-variable transports used by embedded runtimes. pub fn encode_driver_spec_base64url( spec: Option<&crate::proto::compute::v1::DriverSandboxSpec>, ) -> Result { From 29f36b0eaec6a5f8614a8931a81bd7ef61822508 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 20 Aug 2026 21:46:20 -0700 Subject: [PATCH 4/9] fix(compute): complete server driver decoupling Signed-off-by: Drew Newberry --- .../openshell-driver-kubernetes/src/driver.rs | 105 ++++++++---------- crates/openshell-gateway/src/vm.rs | 12 +- crates/openshell-server/Cargo.toml | 3 +- .../src/compute/driver_config.rs | 6 +- crates/openshell-server/src/compute/mod.rs | 55 ++------- crates/openshell-server/src/lib.rs | 30 ++--- proto/compute_driver.proto | 4 + 7 files changed, 79 insertions(+), 136 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 484cd07708..fdedc186d8 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -3864,9 +3864,14 @@ fn sandbox_template_to_k8s_with_validated_config( } apply_pod_driver_config(&mut spec, &driver_config.pod); - // Per-sandbox platform_config.host_users overrides the cluster-wide default. - let use_user_namespaces = platform_config_bool(template, "host_users") - .map_or(params.enable_user_namespaces, |host_users| !host_users); + // Per-sandbox portable intent overrides the cluster-wide default. This + // driver owns the Kubernetes-specific `hostUsers` translation. Accept the + // former platform_config encoding during rolling upgrades from gateways + // that predate the typed field. + let use_user_namespaces = template + .user_namespaces + .or_else(|| platform_config_bool(template, "host_users").map(|host_users| !host_users)) + .unwrap_or(params.enable_user_namespaces); if use_user_namespaces { spec.insert("hostUsers".to_string(), serde_json::json!(false)); @@ -4482,7 +4487,7 @@ fn platform_config_bool(template: &SandboxTemplate, key: &str) -> Option { let config = template.platform_config.as_ref()?; let value = config.fields.get(key)?; match value.kind.as_ref() { - Some(prost_types::value::Kind::BoolValue(b)) => Some(*b), + Some(prost_types::value::Kind::BoolValue(value)) => Some(*value), _ => None, } } @@ -7610,15 +7615,7 @@ mod tests { #[test] fn user_namespaces_per_sandbox_override_enables() { let template = SandboxTemplate { - platform_config: Some(Struct { - fields: std::iter::once(( - "host_users".to_string(), - Value { - kind: Some(Kind::BoolValue(false)), - }, - )) - .collect(), - }), + user_namespaces: Some(true), ..SandboxTemplate::default() }; @@ -7634,7 +7631,7 @@ mod tests { assert_eq!( pod_template["spec"]["hostUsers"], serde_json::json!(false), - "per-sandbox host_users: false must enable user namespaces" + "per-sandbox user namespace intent must set hostUsers: false" ); let caps = pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"] .as_array() @@ -7645,15 +7642,7 @@ mod tests { #[test] fn user_namespaces_per_sandbox_override_disables() { let template = SandboxTemplate { - platform_config: Some(Struct { - fields: std::iter::once(( - "host_users".to_string(), - Value { - kind: Some(Kind::BoolValue(true)), - }, - )) - .collect(), - }), + user_namespaces: Some(false), ..SandboxTemplate::default() }; @@ -7671,7 +7660,7 @@ mod tests { assert!( pod_template["spec"]["hostUsers"].is_null(), - "per-sandbox host_users: true must disable user namespaces even when cluster default is on" + "per-sandbox user namespace intent must override the cluster default" ); let caps = pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"] .as_array() @@ -7683,6 +7672,37 @@ mod tests { ); } + #[test] + fn user_namespaces_accepts_legacy_host_users_encoding() { + let template = SandboxTemplate { + platform_config: Some(Struct { + fields: std::iter::once(( + "host_users".to_string(), + Value { + kind: Some(Kind::BoolValue(false)), + }, + )) + .collect(), + }), + ..SandboxTemplate::default() + }; + + let params = SandboxPodParams::default(); + let pod_template = sandbox_template_to_k8s( + &template, + false, + &std::collections::HashMap::new(), + true, + ¶ms, + ); + + assert_eq!( + pod_template["spec"]["hostUsers"], + serde_json::json!(false), + "legacy host_users: false must still enable user namespaces" + ); + } + #[test] fn automount_service_account_token_is_disabled() { let pod_template = { @@ -7866,43 +7886,6 @@ mod tests { ); } - #[test] - fn platform_config_bool_extracts_value() { - let template = SandboxTemplate { - platform_config: Some(Struct { - fields: std::iter::once(( - "my_bool".to_string(), - Value { - kind: Some(Kind::BoolValue(true)), - }, - )) - .collect(), - }), - ..SandboxTemplate::default() - }; - - assert_eq!(platform_config_bool(&template, "my_bool"), Some(true)); - assert_eq!(platform_config_bool(&template, "missing"), None); - } - - #[test] - fn platform_config_bool_returns_none_for_non_bool() { - let template = SandboxTemplate { - platform_config: Some(Struct { - fields: std::iter::once(( - "a_string".to_string(), - Value { - kind: Some(Kind::StringValue("hello".to_string())), - }, - )) - .collect(), - }), - ..SandboxTemplate::default() - }; - - assert_eq!(platform_config_bool(&template, "a_string"), None); - } - #[test] fn log_level_propagates_as_env_var_to_sandbox_pod() { let spec = SandboxSpec { diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index 69fbdebcbf..e86de28c12 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -20,14 +20,10 @@ //! [`openshell_core::Config`] so the shared core stays free of driver-specific //! plumbing. //! -//! TODO(driver-abstraction): this module still assumes the concrete VM driver -//! (argv shape, guest-TLS flags, libkrun-specific settings). Once we land the -//! generalized compute-driver interface, the CLI-arg plumbing below should -//! be replaced with a driver-agnostic launcher that speaks gRPC to -//! configure the driver — and this file should collapse to the types that -//! are genuinely VM-specific (libkrun log level, vCPU / memory shape) plus a -//! trait implementation registering the VM driver against the generic -//! interface. +//! Process launch remains deliberately VM-specific at this binary composition +//! boundary: it translates gateway configuration into the standalone driver's +//! argv and then connects through the same public compute-driver RPC interface +//! used by operator-managed external drivers. #[cfg(unix)] use hyper_util::rt::TokioIo; diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 0f2bc8abb0..898eef334d 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -27,7 +27,8 @@ openshell-router = { path = "../openshell-router" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } -# Kubernetes client (used by the `generate-certs` subcommand) +# Kubernetes client used by ServiceAccount bootstrap authentication and the +# `generate-certs` subcommand. kube = { workspace = true } k8s-openapi = { workspace = true } diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index 779fcd4eed..32e0a14813 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -3,9 +3,9 @@ //! Selected compute-driver config construction. //! -//! This module owns loading the selected driver config from TOML, applying -//! driver-specific environment overrides, and applying gateway startup defaults. -//! It does not acquire, connect to, or start compute drivers. +//! This module owns loading the selected driver config from TOML and applying +//! gateway startup defaults and endpoint overrides. It does not acquire, +//! connect to, or start compute drivers. use crate::config_file; use crate::defaults::LocalTlsPaths; diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index aca1b04f72..b8f13272fe 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -3765,6 +3765,7 @@ fn driver_sandbox_template_from_public( resources: extract_typed_resources(&template.resources), platform_config: build_platform_config(template), driver_config: select_driver_config(&template.driver_config, driver_name)?, + user_namespaces: template.user_namespaces, }) } @@ -3870,19 +3871,6 @@ fn build_platform_config(template: &SandboxTemplate) -> Option { - let instance = registration - .factory - .build(ComputeDriverBuildContext { - driver_name: registration.name.clone(), - gateway_bind_address: config.bind_address, - gateway_log_level: &config.log_level, - driver_startup, - shutdown_rx, - inherited_config_keys: registration.inherited_config_keys, - }) - .await?; - match instance { + let build_context = ComputeDriverBuildContext { + driver_name: registration.name.clone(), + gateway_bind_address: config.bind_address, + gateway_log_level: &config.log_level, + driver_startup, + shutdown_rx, + inherited_config_keys: registration.inherited_config_keys, + }; + let instance = registration.factory.build(build_context).await?; + let runtime = match instance { ComputeDriverInstance::InProcess(driver) => ComputeRuntime::from_driver( registration.name, driver, @@ -1458,7 +1456,8 @@ async fn build_compute_runtime( Error::execution(format!("failed to create compute runtime: {error}")) })? } - } + }; + runtime } ConfiguredComputeDriver::Remote { name } => { let remote_config = @@ -1471,7 +1470,7 @@ async fn build_compute_runtime( let endpoint = compute::connect_remote_compute_driver(name, &remote_config.socket_path) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - ComputeRuntime::new_remote_driver( + let runtime = ComputeRuntime::new_remote_driver( endpoint, store, sandbox_index, @@ -1480,7 +1479,8 @@ async fn build_compute_runtime( supervisor_sessions, ) .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))? + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + runtime } }; diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index eff5cfa5ea..16a63c0db5 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -214,6 +214,10 @@ message DriverSandboxTemplate { // This is the inner block selected from public SandboxTemplate.driver_config. // The selected driver owns nested schema validation. google.protobuf.Struct driver_config = 12; + // Enable Linux user namespace isolation for the sandbox workload. Drivers + // map this portable intent to their compute platform; when unset, the + // driver's configured default applies. + optional bool user_namespaces = 13; } // Typed compute-resource requirements. From 87fb4f558abec4c3e2e7b53307c91e4ccee40568 Mon Sep 17 00:00:00 2001 From: Drew Newberry <385+drew@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:05:43 +0000 Subject: [PATCH 5/9] fix(compute): preserve driver integrations after rebase Signed-off-by: Drew Newberry <385+drew@users.noreply.github.com> --- crates/openshell-core/src/lib.rs | 1 + crates/openshell-driver-docker/src/lib.rs | 295 +++++++++++++++++++- crates/openshell-gateway/src/lib.rs | 55 +++- crates/openshell-server/src/otel_tracing.rs | 2 +- 4 files changed, 349 insertions(+), 4 deletions(-) diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index f4170002c7..9aecfcb1f1 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -45,6 +45,7 @@ pub mod provider_credentials; pub mod sandbox_env; pub mod secrets; pub mod settings; +pub mod spiffe; pub mod telemetry; pub mod time; pub mod transport_errors; diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 6a46af4bef..1d70d6697b 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5,6 +5,8 @@ #![allow(clippy::result_large_err)] +pub mod otel_tracing; + use bollard::Docker; use bollard::errors::Error as BollardError; use bollard::models::{ @@ -54,10 +56,12 @@ use openshell_core::proto_struct::{ }; use openshell_core::{Error, Result as CoreResult}; use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; +use std::task::{Context, Poll}; use std::time::Duration; use tokio::sync::{Mutex, broadcast, mpsc}; use tokio::task::JoinHandle; @@ -385,6 +389,119 @@ fn default_true() -> bool { type WatchStream = Pin> + Send + 'static>>; +struct TracedWatchStream { + inner: WatchStream, + span: tracing::Span, + finished: bool, +} + +impl Stream for TracedWatchStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let span = self.span.clone(); + let _entered = span.enter(); + let result = self.inner.as_mut().poll_next(cx); + if !self.finished { + match &result { + Poll::Ready(Some(Err(status))) => { + openshell_otel::mark_error(&self.span); + self.span + .record("rpc.grpc.status_code", status.code() as i32); + self.finished = true; + } + Poll::Ready(None) => { + self.span + .record("rpc.grpc.status_code", tonic::Code::Ok as i32); + self.finished = true; + } + Poll::Pending | Poll::Ready(Some(Ok(_))) => {} + } + } + result + } +} + +impl Drop for TracedWatchStream { + fn drop(&mut self) { + if !self.finished { + openshell_otel::mark_error(&self.span); + self.span + .record("rpc.grpc.status_code", tonic::Code::Cancelled as i32); + } + } +} + +/// Compute-driver service wrapper that preserves the standalone RPC trace +/// boundary while Docker runs in the gateway process. +#[derive(Clone)] +pub struct ComputeDriverService { + driver: DockerComputeDriver, + trace_in_process_rpc: bool, +} + +impl ComputeDriverService { + #[must_use] + pub fn new(driver: DockerComputeDriver) -> Self { + Self { + driver, + trace_in_process_rpc: false, + } + } + + #[must_use] + pub fn new_in_process(driver: DockerComputeDriver) -> Self { + Self { + driver, + trace_in_process_rpc: true, + } + } + + fn in_process_rpc_span( + &self, + operation: &'static str, + method: &'static str, + ) -> Option { + self.trace_in_process_rpc.then(|| { + tracing::info_span!( + target: "openshell_driver_docker::otel_tracing", + "driver_rpc", + otel.name = operation, + otel.kind = "server", + otel.status_code = tracing::field::Empty, + rpc.system = "grpc", + rpc.service = "openshell.compute.v1.ComputeDriver", + rpc.method = method, + rpc.grpc.status_code = tracing::field::Empty, + ) + }) + } + + async fn trace_rpc( + &self, + operation: &'static str, + method: &'static str, + future: impl Future>, + ) -> Result { + use tracing::Instrument as _; + + let Some(span) = self.in_process_rpc_span(operation, method) else { + return future.await; + }; + let result = future.instrument(span.clone()).await; + match &result { + Ok(_) => { + span.record("rpc.grpc.status_code", tonic::Code::Ok as i32); + } + Err(status) => { + openshell_otel::mark_error(&span); + span.record("rpc.grpc.status_code", status.code() as i32); + } + } + result + } +} + /// Return the first responsive local Docker API socket. #[must_use] pub fn detect_socket() -> Option { @@ -1563,12 +1680,188 @@ impl DockerComputeDriver { } } +#[tonic::async_trait] +impl ComputeDriver for ComputeDriverService { + type WatchSandboxesStream = WatchStream; + + async fn authenticate_sandbox( + &self, + request: Request, + ) -> Result, Status> + { + self.trace_rpc( + "driver.authenticate_sandbox", + "authenticate_sandbox", + ComputeDriver::authenticate_sandbox(&self.driver, request), + ) + .await + } + + async fn get_capabilities( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.get_capabilities", + "get_capabilities", + ComputeDriver::get_capabilities(&self.driver, request), + ) + .await + } + + async fn get_gateway_listener_requirements( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.get_gateway_listener_requirements", + "get_gateway_listener_requirements", + ComputeDriver::get_gateway_listener_requirements(&self.driver, request), + ) + .await + } + + async fn validate_sandbox_create( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.validate_sandbox_create", + "validate_sandbox_create", + ComputeDriver::validate_sandbox_create(&self.driver, request), + ) + .await + } + + async fn get_sandbox( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.get_sandbox", + "get_sandbox", + ComputeDriver::get_sandbox(&self.driver, request), + ) + .await + } + + async fn list_sandboxes( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.list_sandboxes", + "list_sandboxes", + ComputeDriver::list_sandboxes(&self.driver, request), + ) + .await + } + + async fn create_sandbox( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.create_sandbox", + "create_sandbox", + ComputeDriver::create_sandbox(&self.driver, request), + ) + .await + } + + async fn stop_sandbox( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.stop_sandbox", + "stop_sandbox", + ComputeDriver::stop_sandbox(&self.driver, request), + ) + .await + } + + async fn start_sandbox( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.start_sandbox", + "start_sandbox", + ComputeDriver::start_sandbox(&self.driver, request), + ) + .await + } + + async fn delete_sandbox( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.delete_sandbox", + "delete_sandbox", + ComputeDriver::delete_sandbox(&self.driver, request), + ) + .await + } + + async fn watch_sandboxes( + &self, + request: Request, + ) -> Result, Status> { + use tracing::Instrument as _; + + let create_stream = ComputeDriver::watch_sandboxes(&self.driver, request); + let Some(span) = self.in_process_rpc_span("driver.watch_sandboxes", "watch_sandboxes") + else { + return create_stream.await; + }; + match create_stream.instrument(span.clone()).await { + Ok(response) => Ok(Response::new(Box::pin(TracedWatchStream { + inner: response.into_inner(), + span, + finished: false, + }))), + Err(status) => { + openshell_otel::mark_error(&span); + span.record("rpc.grpc.status_code", status.code() as i32); + Err(status) + } + } + } + + async fn ensure_workspace( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.ensure_workspace", + "ensure_workspace", + ComputeDriver::ensure_workspace(&self.driver, request), + ) + .await + } + + async fn delete_workspace( + &self, + request: Request, + ) -> Result, Status> { + self.trace_rpc( + "driver.delete_workspace", + "delete_workspace", + ComputeDriver::delete_workspace(&self.driver, request), + ) + .await + } +} + #[tonic::async_trait] impl ComputeDriver for DockerComputeDriver { async fn authenticate_sandbox( &self, _request: Request, - ) -> Result, Status> { + ) -> Result, Status> + { Err(Status::unimplemented( "docker does not authenticate sandbox credentials", )) diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index a573a18aed..6a5b503b29 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -48,6 +48,7 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { registration .with_telemetry_category(TelemetryComputeDriver::anonymous_category("kubernetes")) .without_mtls_user_auth() + .with_tracing_setup(kubernetes_tracing_setup) .with_inherited_config_keys(&[ "namespace", "default_image", @@ -89,6 +90,7 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { registration .with_telemetry_category(TelemetryComputeDriver::anonymous_category("docker")) .with_local_singleplayer() + .with_tracing_setup(docker_tracing_setup) .with_inherited_config_keys(&[ "sandbox_namespace", "default_image", @@ -117,6 +119,30 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { } } +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn kubernetes_tracing_setup( + otlp_endpoint: Option<&str>, +) -> openshell_server::ComputeDriverTracingSetup { + let (provider, error) = openshell_driver_kubernetes::otel_tracing::provider_for(otlp_endpoint); + let layer = provider.as_ref().map(|provider| { + let layer: openshell_server::ComputeDriverTracingLayer = Box::new( + openshell_driver_kubernetes::otel_tracing::in_process_layer(provider), + ); + layer + }); + let shutdown = provider.map(|provider| { + let shutdown: openshell_server::ComputeDriverTracingShutdown = + Box::new(move || provider.shutdown().map_err(|error| error.to_string())); + shutdown + }); + openshell_server::ComputeDriverTracingSetup::new( + layer, + shutdown, + error.map(|error| error.to_string()), + Some(openshell_driver_kubernetes::otel_tracing::IN_PROCESS_TARGET_PREFIX), + ) +} + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] fn podman_tracing_setup( otlp_endpoint: Option<&str>, @@ -141,6 +167,30 @@ fn podman_tracing_setup( ) } +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn docker_tracing_setup( + otlp_endpoint: Option<&str>, +) -> openshell_server::ComputeDriverTracingSetup { + let (provider, error) = openshell_driver_docker::otel_tracing::provider_for(otlp_endpoint); + let layer = provider.as_ref().map(|provider| { + let layer: openshell_server::ComputeDriverTracingLayer = Box::new( + openshell_driver_docker::otel_tracing::in_process_layer(provider), + ); + layer + }); + let shutdown = provider.map(|provider| { + let shutdown: openshell_server::ComputeDriverTracingShutdown = + Box::new(move || provider.shutdown().map_err(|error| error.to_string())); + shutdown + }); + openshell_server::ComputeDriverTracingSetup::new( + layer, + shutdown, + error.map(|error| error.to_string()), + Some(openshell_driver_docker::otel_tracing::IN_PROCESS_TARGET_PREFIX), + ) +} + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct KubernetesFactory; @@ -166,7 +216,7 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { ) .await .map_err(|error| openshell_core::Error::execution(error.to_string()))?; - let driver = openshell_driver_kubernetes::ComputeDriverService::new(driver); + let driver = openshell_driver_kubernetes::ComputeDriverService::new_in_process(driver); Ok(openshell_server::ComputeDriverInstance::InProcess( std::sync::Arc::new(driver), )) @@ -198,6 +248,7 @@ impl openshell_server::ComputeDriverFactory for DockerFactory { ) .await .map_err(|error| openshell_core::Error::execution(error.to_string()))?; + let driver = openshell_driver_docker::ComputeDriverService::new_in_process(driver); Ok(openshell_server::ComputeDriverInstance::InProcess( std::sync::Arc::new(driver), )) @@ -235,7 +286,7 @@ impl openshell_server::ComputeDriverFactory for PodmanFactory { let driver = openshell_driver_podman::PodmanComputeDriver::new(config) .await .map_err(|error| openshell_core::Error::execution(error.to_string()))?; - let driver = openshell_driver_podman::ComputeDriverService::new(driver); + let driver = openshell_driver_podman::ComputeDriverService::new_in_process(driver); Ok(openshell_server::ComputeDriverInstance::InProcess( std::sync::Arc::new(driver), )) diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index eec3127651..fc9afe3d18 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -148,7 +148,7 @@ where openshell_otel::layer_excluding_target_prefix( provider, INSTRUMENTATION_SCOPE, - excluded_target_prefix.unwrap_or("\0"), + excluded_target_prefix, ) } From 0c829d092eb6580ad8f3cde3311691e6b60383f1 Mon Sep 17 00:00:00 2001 From: Drew Newberry <385+drew@users.noreply.github.com> Date: Thu, 27 Aug 2026 06:44:04 +0000 Subject: [PATCH 6/9] fix(compute): preserve docker tracing after decoupling Signed-off-by: Drew Newberry <385+drew@users.noreply.github.com> --- crates/openshell-driver-docker/src/lib.rs | 192 ++++++++++++++++++---- crates/openshell-server/src/lib.rs | 10 +- 2 files changed, 160 insertions(+), 42 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 1d70d6697b..d218593cf3 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -55,6 +55,7 @@ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; use openshell_core::{Error, Result as CoreResult}; +use opentelemetry::trace::TraceContextExt as _; use std::collections::{HashMap, HashSet}; use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -67,7 +68,8 @@ use tokio::sync::{Mutex, broadcast, mpsc}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; -use tracing::{debug, info, warn}; +use tracing::{Instrument as _, debug, info, warn}; +use tracing_opentelemetry::OpenTelemetrySpanExt as _; use url::Url; const WATCH_BUFFER: usize = 128; @@ -84,6 +86,28 @@ const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; const DOCKER_NETWORK_DRIVER: &str = "bridge"; +fn provisioning_span( + parent: &opentelemetry::Context, + sandbox: &DriverSandbox, + image_ref: &str, +) -> tracing::Span { + let span = tracing::info_span!( + parent: None, + "docker.provision", + otel.name = "docker.provision", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + sandbox.name = %sandbox.name, + image.ref = %image_ref, + ); + let parent_span_context = parent.span().span_context().clone(); + if parent_span_context.is_valid() { + let parent = opentelemetry::Context::new().with_remote_span_context(parent_span_context); + let _ = span.set_parent(parent); + } + span +} + /// Gateway-local configuration for the Docker compute driver. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] #[serde(default, deny_unknown_fields)] @@ -904,7 +928,7 @@ impl DockerComputeDriver { &sandbox.id, "Scheduled", format!("Docker sandbox accepted for image \"{image}\""), - HashMap::from([("image_ref".to_string(), image)]), + HashMap::from([("image_ref".to_string(), image.clone())]), ); self.publish_sandbox_snapshot(pending_sandbox_snapshot( sandbox, @@ -916,9 +940,14 @@ impl DockerComputeDriver { let driver = self.clone(); let sandbox_for_task = sandbox.clone(); let sandbox_id = sandbox.id.clone(); - let task = tokio::spawn(async move { - driver.provision_sandbox(sandbox_for_task).await; - }); + let parent = tracing::Span::current().context(); + let provisioning_span = provisioning_span(&parent, sandbox, &image); + let task = tokio::spawn( + async move { + driver.provision_sandbox(sandbox_for_task).await; + } + .instrument(provisioning_span), + ); let mut pending = self.pending.lock().await; if let Some(record) = pending.get_mut(&sandbox_id) { @@ -941,20 +970,42 @@ impl DockerComputeDriver { } } + #[tracing::instrument( + name = "docker.provision_sandbox", + skip(self, sandbox), + fields( + otel.name = "docker.provision_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + sandbox.name = %sandbox.name, + ) + )] async fn provision_sandbox_inner( &self, sandbox: &DriverSandbox, ) -> Result<(), DockerProvisioningFailure> { + let span_status = openshell_otel::ErrorStatusGuard::current(); let validated = Self::validated_sandbox(sandbox, &self.config).map_err(|status| { DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; let template = validated.template; - let image = self - .ensure_image_available(&sandbox.id, &template.image) - .await - .map_err(|status| { - DockerProvisioningFailure::new("ImagePullFailed", status.message()) - })?; + let image = async { + openshell_otel::record_error_result( + self.ensure_image_available(&sandbox.id, &template.image) + .await + .map_err(|status| { + DockerProvisioningFailure::new("ImagePullFailed", status.message()) + }), + ) + } + .instrument(tracing::info_span!( + "docker.prepare_image", + otel.name = "docker.prepare_image", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + image.ref = %template.image, + )) + .await?; let token_file_created = write_sandbox_token_file(sandbox, &self.config) .await .map_err(|status| { @@ -988,25 +1039,37 @@ impl DockerComputeDriver { } DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; - self.docker - .create_container( - Some( - CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(), - ), - create_body, + async { + openshell_otel::record_error_result( + self.docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(container_name.as_str()) + .build(), + ), + create_body, + ) + .await + .map_err(|err| { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + DockerProvisioningFailure::from_status( + "ContainerCreateFailed", + create_status_from_docker_error("create docker sandbox container", err), + ) + }), ) - .await - .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::from_status( - "ContainerCreateFailed", - create_status_from_docker_error("create docker sandbox container", err), - ) - })?; + } + .instrument(tracing::info_span!( + "docker.create_container", + otel.name = "docker.create_container", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + container.name = %container_name, + )) + .await?; self.publish_docker_progress( &sandbox.id, "Created", @@ -1014,7 +1077,20 @@ impl DockerComputeDriver { HashMap::from([("container_name".to_string(), container_name.clone())]), ); - if let Err(err) = self.docker.start_container(&container_name, None).await { + let start_result = async { + openshell_otel::record_error_result( + self.docker.start_container(&container_name, None).await, + ) + } + .instrument(tracing::info_span!( + "docker.start_container", + otel.name = "docker.start_container", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox.id, + container.name = %container_name, + )) + .await; + if let Err(err) = start_result { let cleanup = self .docker .remove_container( @@ -1055,7 +1131,7 @@ impl DockerComputeDriver { ); } - Ok(()) + span_status.finish(Ok(())) } async fn delete_sandbox_inner( @@ -1169,17 +1245,29 @@ impl DockerComputeDriver { /// Returns `Ok(true)` when a container existed and was started (or was /// already running), `Ok(false)` when no managed container is found for /// the sandbox, and `Err(...)` for any Docker failure. + #[tracing::instrument( + name = "docker.start_sandbox", + skip(self), + fields( + otel.name = "docker.start_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + sandbox.name = %sandbox_name, + ) + )] pub async fn start_sandbox( &self, sandbox_id: &str, sandbox_name: &str, ) -> Result { + let span_status = openshell_otel::ErrorStatusGuard::current(); + require_sandbox_identifier(sandbox_id, sandbox_name)?; self.lifecycle_event_fences.begin_start(sandbox_id); let result = self .start_sandbox_with_lifecycle_fence(sandbox_id, sandbox_name) .await; self.lifecycle_event_fences.finish_start(sandbox_id); - result + span_status.finish(result) } async fn start_sandbox_with_lifecycle_fence( @@ -1951,22 +2039,44 @@ impl ComputeDriver for DockerComputeDriver { })) } + #[tracing::instrument( + name = "docker.schedule_sandbox", + skip(self, request), + fields( + otel.name = "docker.schedule_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %request.get_ref().sandbox.as_ref().map_or("", |sandbox| sandbox.id.as_str()), + sandbox.name = %request.get_ref().sandbox.as_ref().map_or("", |sandbox| sandbox.name.as_str()), + ) + )] async fn create_sandbox( &self, request: Request, ) -> Result, Status> { + let span_status = openshell_otel::ErrorStatusGuard::current(); let sandbox = request .into_inner() .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; self.create_sandbox_inner(&sandbox).await?; - Ok(Response::new(CreateSandboxResponse {})) + span_status.finish(Ok(Response::new(CreateSandboxResponse {}))) } + #[tracing::instrument( + name = "docker.stop_sandbox", + skip(self, request), + fields( + otel.name = "docker.stop_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %request.get_ref().sandbox_id, + sandbox.name = %request.get_ref().sandbox_name, + ) + )] async fn stop_sandbox( &self, request: Request, ) -> Result, Status> { + let span_status = openshell_otel::ErrorStatusGuard::current(); let request = request.into_inner(); require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; @@ -1974,7 +2084,7 @@ impl ComputeDriver for DockerComputeDriver { .await?; self.publish_container_snapshot(&request.sandbox_id, &request.sandbox_name) .await?; - Ok(Response::new(StopSandboxResponse {})) + span_status.finish(Ok(Response::new(StopSandboxResponse {}))) } async fn start_sandbox( @@ -1982,7 +2092,6 @@ impl ComputeDriver for DockerComputeDriver { request: Request, ) -> Result, Status> { let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; if !Self::start_sandbox(self, &request.sandbox_id, &request.sandbox_name).await? { return Err(Status::not_found("sandbox not found")); } @@ -1991,10 +2100,21 @@ impl ComputeDriver for DockerComputeDriver { Ok(Response::new(StartSandboxResponse {})) } + #[tracing::instrument( + name = "docker.delete_sandbox", + skip(self, request), + fields( + otel.name = "docker.delete_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %request.get_ref().sandbox_id, + sandbox.name = %request.get_ref().sandbox_name, + ) + )] async fn delete_sandbox( &self, request: Request, ) -> Result, Status> { + let span_status = openshell_otel::ErrorStatusGuard::current(); let request = request.into_inner(); require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; @@ -2013,7 +2133,7 @@ impl ComputeDriver for DockerComputeDriver { }); } - Ok(Response::new(DeleteSandboxResponse { deleted })) + span_status.finish(Ok(Response::new(DeleteSandboxResponse { deleted }))) } async fn watch_sandboxes( diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 6edd402cc3..ba56376ac8 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1426,7 +1426,7 @@ async fn build_compute_runtime( inherited_config_keys: registration.inherited_config_keys, }; let instance = registration.factory.build(build_context).await?; - let runtime = match instance { + match instance { ComputeDriverInstance::InProcess(driver) => ComputeRuntime::from_driver( registration.name, driver, @@ -1456,8 +1456,7 @@ async fn build_compute_runtime( Error::execution(format!("failed to create compute runtime: {error}")) })? } - }; - runtime + } } ConfiguredComputeDriver::Remote { name } => { let remote_config = @@ -1470,7 +1469,7 @@ async fn build_compute_runtime( let endpoint = compute::connect_remote_compute_driver(name, &remote_config.socket_path) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - let runtime = ComputeRuntime::new_remote_driver( + ComputeRuntime::new_remote_driver( endpoint, store, sandbox_index, @@ -1479,8 +1478,7 @@ async fn build_compute_runtime( supervisor_sessions, ) .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - runtime + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))? } }; From d5be8a8c997324c0de5463349d27eeae23cf4c2f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 27 Aug 2026 19:28:30 -0700 Subject: [PATCH 7/9] fix(compute): preserve driver behavior after extraction Signed-off-by: Drew Newberry --- .../build-openshell-mxc-windows/SKILL.md | 10 +- .../build-openshell-mxc-windows/reference.md | 4 +- .github/workflows/branch-e2e.yml | 2 +- .github/workflows/build-gateway-binaries.yml | 2 +- AGENTS.md | 1 + Cargo.lock | 29 +- architecture/windows-msvc-build.md | 17 +- crates/openshell-driver-docker/src/main.rs | 2 +- crates/openshell-driver-podman/src/driver.rs | 23 +- crates/openshell-driver-podman/src/lib.rs | 1 + .../src/socket_discovery.rs | 402 ++++++++++++++++++ crates/openshell-driver-vm/Cargo.toml | 1 + crates/openshell-driver-vm/src/driver.rs | 23 +- crates/openshell-gateway/Cargo.toml | 4 + crates/openshell-gateway/src/lib.rs | 119 +++++- .../src/compute/driver_config.rs | 13 - crates/openshell-server/src/compute/mod.rs | 98 ++++- crates/openshell-server/src/grpc/policy.rs | 28 +- crates/openshell-server/src/lib.rs | 36 +- crates/openshell-server/src/otel_tracing.rs | 5 - crates/openshell-server/src/tracing_setup.rs | 4 +- docs/reference/sandbox-compute-drivers.mdx | 6 +- tasks/scripts/windows-msvc.ps1 | 4 +- 23 files changed, 705 insertions(+), 129 deletions(-) create mode 100644 crates/openshell-driver-podman/src/socket_discovery.rs diff --git a/.agents/skills/build-openshell-mxc-windows/SKILL.md b/.agents/skills/build-openshell-mxc-windows/SKILL.md index d61fc555e9..954fcedf2f 100644 --- a/.agents/skills/build-openshell-mxc-windows/SKILL.md +++ b/.agents/skills/build-openshell-mxc-windows/SKILL.md @@ -231,7 +231,7 @@ crypto dependency builds. | `windows:build:arm64` | Release-builds `openshell-gateway.exe` and `openshell.exe` for ARM64. | | `windows:test:x64` | Runs native x64 workspace tests with `--no-fail-fast`, excluding unsupported Windows packages as top-level workspace targets. | | `windows:test:arm64` | Runs native ARM64 workspace tests with `--no-fail-fast` and the same package exclusions. Rejects non-ARM64 hosts. | -| `windows:test:unsupported:x64` | Re-runs focused `openshell-server` tests for unsupported Windows driver behavior. | +| `windows:test:unsupported:x64` | Re-runs focused `openshell-gateway` tests for unsupported Windows driver behavior. | | `windows:test:unsupported:arm64` | Re-runs the same focused contracts natively on ARM64. Rejects non-ARM64 hosts. | | `windows:artifacts` | Reports size and SHA256 for release artifacts that exist. | | `windows:ci` | Runs the full ordered x64-host Windows CI lane, plus ARM64 check/build when not skipped. | @@ -248,10 +248,10 @@ Windows must continue to reject unsupported compute drivers clearly. | Driver | Windows build behavior | Runtime behavior | |---|---|---| -| Docker | Driver crate excluded; server config contract retained. | Gateway construction returns unsupported. | -| Kubernetes | Driver crate excluded; server config contract retained. | Gateway construction returns unsupported. | -| Podman | Driver crate excluded; server config contract retained. | Gateway construction returns unsupported. | -| VM | Driver crate excluded from workspace validation. | Gateway construction returns unsupported. | +| Docker | Driver crate excluded; gateway registration stub retained. | Gateway construction returns unsupported. | +| Kubernetes | Driver crate excluded; gateway registration stub retained. | Gateway construction returns unsupported. | +| Podman | Driver crate excluded; gateway registration stub retained. | Gateway construction returns unsupported. | +| VM | Driver crate excluded; gateway registration stub retained. | Gateway construction returns unsupported. | The focused contract tasks for either native architecture run: diff --git a/.agents/skills/build-openshell-mxc-windows/reference.md b/.agents/skills/build-openshell-mxc-windows/reference.md index 4a5f2668a1..16b9a48586 100644 --- a/.agents/skills/build-openshell-mxc-windows/reference.md +++ b/.agents/skills/build-openshell-mxc-windows/reference.md @@ -85,7 +85,7 @@ Windows is a build target only. These runtimes remain unsupported: Rules: -- Keep config/library stubs where the gateway needs them. +- Keep registration stubs in the gateway composition crate where the gateway needs them. - Return clear unsupported errors at runtime. - Do not build standalone Windows driver binaries. - Do not add Docker Desktop, WSL, Hyper-V, Podman machine, Podman Desktop, or @@ -180,7 +180,7 @@ blocked dependency. ### Focused tests report many filtered-out tests This is expected for `windows:test:unsupported:x64`. Cargo runs one named test -and filters the other `openshell-server` tests. Report these as filtered, not +and filters the other `openshell-gateway` tests. Report these as filtered, not ignored. ## Reporting Counts diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 580a05c1f9..2e316f88bd 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -166,7 +166,7 @@ jobs: interpreter: /lib/ld-linux-aarch64.so.1 uses: ./.github/workflows/build-binaries.yml with: - package: openshell-server + package: openshell-gateway binary: openshell-gateway artifact-name: openshell-gateway-plain-${{ matrix.triple }} triple: ${{ matrix.triple }} diff --git a/.github/workflows/build-gateway-binaries.yml b/.github/workflows/build-gateway-binaries.yml index ed5219e7c6..391e421080 100644 --- a/.github/workflows/build-gateway-binaries.yml +++ b/.github/workflows/build-gateway-binaries.yml @@ -43,7 +43,7 @@ jobs: interpreter: "" uses: ./.github/workflows/build-binaries.yml with: - package: openshell-server + package: openshell-gateway binary: openshell-gateway triple: ${{ matrix.triple }} runner: ${{ matrix.runner }} diff --git a/AGENTS.md b/AGENTS.md index 103e45bafc..33dad248d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-driver-db-credstore/` | Database credential driver | In-process `CredentialDriver` backend for gateway database credential storage | | `crates/openshell-driver-kubernetes/` | Kubernetes compute driver | In-process `ComputeDriver` backend for K8s sandbox pods | | `crates/openshell-driver-docker/` | Docker compute driver | In-process `ComputeDriver` backend for local Docker sandbox containers | +| `crates/openshell-driver-mxc/` | MXC compute driver | Windows in-process `ComputeDriver` backend for MXC sandbox execution | | `crates/openshell-driver-podman/` | Podman compute driver | In-process `ComputeDriver` backend for local Podman sandbox containers | | `crates/openshell-driver-vm/` | VM compute driver | Standalone libkrun-backed `ComputeDriver` subprocess (embeds its own rootfs + runtime) | | `crates/openshell-prover/` | Policy prover | Policy verification and proof generation | diff --git a/Cargo.lock b/Cargo.lock index 9188a93d0c..a4815554a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4069,6 +4069,7 @@ dependencies = [ "nix 0.29.0", "oci-client", "openshell-core", + "openshell-driver-podman", "openshell-otel", "openshell-otel-test-support", "openshell-policy", @@ -4113,6 +4114,30 @@ dependencies = [ "tower 0.5.3", ] +[[package]] +name = "openshell-gateway" +version = "0.0.0" +dependencies = [ + "async-trait", + "hyper-util", + "miette", + "nix 0.29.0", + "openshell-core", + "openshell-driver-docker", + "openshell-driver-kubernetes", + "openshell-driver-mxc", + "openshell-driver-podman", + "openshell-otel", + "openshell-server", + "rustix 1.1.4", + "serde", + "tempfile", + "tokio", + "tonic", + "tower 0.5.3", + "tracing", +] + [[package]] name = "openshell-gateway-interceptors" version = "0.0.0" @@ -4324,11 +4349,7 @@ dependencies = [ "openshell-bootstrap", "openshell-core", "openshell-driver-db-credstore", - "openshell-driver-docker", - "openshell-driver-kubernetes", "openshell-driver-kubernetes-secrets", - "openshell-driver-mxc", - "openshell-driver-podman", "openshell-driver-vault", "openshell-extension-core", "openshell-gateway-interceptors", diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index e940b83273..56927f81f4 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -23,9 +23,10 @@ driver. It does not make Windows a Docker, Kubernetes, Podman, or VM runtime hos ## Unsupported Driver Strategy -The gateway uses platform-specific configuration contracts on Windows. These -contracts preserve config-file parsing and reject unsupported driver selection -with a clear error without depending on the runtime driver crates. +The gateway composition crate installs platform-specific registration stubs on +Windows. These registrations preserve config-file selection and reject +unsupported drivers with a clear error without depending on their runtime +crates. The Windows lane does not build, release, package, or smoke-test standalone driver binaries for Docker, Kubernetes, Podman, or VM. Those binaries are Linux @@ -39,10 +40,10 @@ on Windows. | Driver | Windows build behavior | Runtime behavior | |---|---|---| -| Docker | Driver crate excluded; server config contract retained. | Gateway construction returns unsupported. | -| Kubernetes | Driver crate excluded; server config contract retained. | Gateway construction returns unsupported. | -| Podman | Driver crate excluded; server config contract retained. | Gateway construction returns unsupported. | -| VM | Driver crate excluded from workspace validation. | Gateway construction returns unsupported. | +| Docker | Driver crate excluded; gateway registration stub retained. | Gateway construction returns unsupported. | +| Kubernetes | Driver crate excluded; gateway registration stub retained. | Gateway construction returns unsupported. | +| Podman | Driver crate excluded; gateway registration stub retained. | Gateway construction returns unsupported. | +| VM | Driver crate excluded; gateway registration stub retained. | Gateway construction returns unsupported. | | MXC | Driver links into the native gateway and runs in Windows validation. | `process_container` is default-deny; grant-only `isolation_session` requires explicit configuration. | This keeps Windows behavior explicit without carrying runtime dependencies or @@ -66,7 +67,7 @@ Windows validation is exposed through `tasks/windows.toml`: | `windows:build:arm64` | Build release ARM64 `openshell-gateway.exe` and `openshell.exe`. | | `windows:test:x64` | Run native x64 workspace tests, including MXC mapper and lifecycle tests, while excluding unsupported Windows packages as top-level test targets. | | `windows:test:arm64` | Run native ARM64 workspace tests with the same package exclusions. | -| `windows:test:unsupported:x64` | Run focused server/runtime tests for unsupported driver contracts. | +| `windows:test:unsupported:x64` | Run focused gateway-composition tests for unsupported driver contracts. | | `windows:test:unsupported:arm64` | Run the same focused contracts natively on ARM64. | | `windows:ci` | Run check, build, test, unsupported-contract tests, and artifact reporting. | diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs index 7c4b5b1cb4..11a2762849 100644 --- a/crates/openshell-driver-docker/src/main.rs +++ b/crates/openshell-driver-docker/src/main.rs @@ -6,8 +6,8 @@ use std::path::PathBuf; use clap::Parser; use miette::{IntoDiagnostic, Result}; -use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_core::VERSION; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_docker::otel_tracing::compute_driver_rpc_layer; use openshell_driver_docker::{ComputeDriverService, DockerComputeConfig, DockerComputeDriver}; use tracing::info; diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index f3acd1ebf8..5bc6ea03b3 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -314,28 +314,7 @@ fn podman_gpu_selection_error(err: CdiGpuSelectionError) -> ComputeDriverError { /// Return the first responsive local Podman API socket. #[must_use] pub fn detect_socket() -> Option { - let mut candidates = Vec::new(); - if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") - && !path.trim().is_empty() - { - candidates.push(PathBuf::from(path)); - } - if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { - candidates.push(PathBuf::from(runtime_dir).join("podman/podman.sock")); - } - #[cfg(target_os = "linux")] - candidates.push(PathBuf::from(format!( - "/run/user/{}/podman/podman.sock", - rustix::process::geteuid().as_raw() - ))); - if let Some(home) = std::env::var_os("HOME") { - candidates - .push(PathBuf::from(home).join(".local/share/containers/podman/machine/podman.sock")); - } - openshell_core::local_api_socket::first_responsive_socket(&candidates, |response| { - openshell_core::local_api_socket::http_response_is_success(response) - && openshell_core::local_api_socket::contains_ascii(response, b"Libpod-Api-Version:") - }) + crate::socket_discovery::detect_socket() } #[must_use] diff --git a/crates/openshell-driver-podman/src/lib.rs b/crates/openshell-driver-podman/src/lib.rs index 7a78cf5d92..115e64eb2f 100644 --- a/crates/openshell-driver-podman/src/lib.rs +++ b/crates/openshell-driver-podman/src/lib.rs @@ -7,6 +7,7 @@ pub(crate) mod container; pub mod driver; pub mod grpc; pub mod otel_tracing; +mod socket_discovery; #[cfg(test)] pub(crate) mod test_utils; pub(crate) mod watcher; diff --git a/crates/openshell-driver-podman/src/socket_discovery.rs b/crates/openshell-driver-podman/src/socket_discovery.rs new file mode 100644 index 0000000000..e7a6c79e74 --- /dev/null +++ b/crates/openshell-driver-podman/src/socket_discovery.rs @@ -0,0 +1,402 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Podman socket discovery for local and machine-backed installations. + +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +const DISCOVERY_TIMEOUT: Duration = Duration::from_secs(5); +const DISCOVERY_POLL_INTERVAL: Duration = Duration::from_millis(25); + +/// Return the first responsive well-known socket, then ask the Podman CLI for +/// the active native or machine-backed connection. +pub fn detect_socket() -> Option { + openshell_core::local_api_socket::first_responsive_socket(&socket_candidates(), |response| { + openshell_core::local_api_socket::http_response_is_success(response) + && openshell_core::local_api_socket::contains_ascii(response, b"Libpod-Api-Version:") + }) + .or_else(discover_socket) +} + +fn socket_candidates() -> Vec { + let mut candidates = Vec::new(); + if let Some(path) = env_var_nonempty("OPENSHELL_PODMAN_SOCKET") { + candidates.push(PathBuf::from(path)); + } + if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { + candidates.push(PathBuf::from(runtime_dir).join("podman/podman.sock")); + } + #[cfg(target_os = "linux")] + candidates.push(PathBuf::from(format!( + "/run/user/{}/podman/podman.sock", + rustix::process::geteuid().as_raw() + ))); + if let Some(home) = std::env::var_os("HOME") { + candidates + .push(PathBuf::from(home).join(".local/share/containers/podman/machine/podman.sock")); + } + candidates +} + +/// Query the same active connection selected by the Podman CLI. This covers +/// named/rootful machines and provider-specific forwarded socket locations that +/// do not have a stable well-known path. +fn discover_socket() -> Option { + let stdout = run_podman_capture(&["info", "--format", "json"])?; + + // A successful `podman info` proves this explicit socket is usable. A named + // connection has higher precedence, so only honor CONTAINER_HOST directly + // when no CONTAINER_CONNECTION is set. + if env_var_nonempty("CONTAINER_CONNECTION").is_none() + && let Some(path) = env_var_nonempty("CONTAINER_HOST") + .as_deref() + .and_then(unix_url_socket_path) + { + return Some(path); + } + + let info: serde_json::Value = serde_json::from_slice(&stdout).ok()?; + if !info["host"]["serviceIsRemote"].as_bool().unwrap_or(false) { + return parse_info_socket(&info); + } + + discover_machine_socket() +} + +fn discover_machine_socket() -> Option { + let connections = connection_list(); + let active = active_machine( + env_var_nonempty("CONTAINER_CONNECTION").as_deref(), + env_var_nonempty("CONTAINER_HOST").as_deref(), + connections.as_ref(), + )?; + machine_inspect_targets(&active) + .into_iter() + .find_map(|name| { + let stdout = run_podman_capture(&["machine", "inspect", &name])?; + let machines: serde_json::Value = serde_json::from_slice(&stdout).ok()?; + parse_machine_socket(&machines) + }) +} + +fn active_machine( + container_connection: Option<&str>, + container_host: Option<&str>, + connections: Option<&serde_json::Value>, +) -> Option { + if let Some(name) = container_connection.filter(|name| !name.trim().is_empty()) { + return Some(name.to_string()); + } + if let Some(host) = container_host.filter(|host| !host.trim().is_empty()) { + // An explicit non-machine endpoint must not fall back to an unrelated + // local machine. + return connection_name_for_uri(connections?, host); + } + default_machine_connection(connections).or_else(|| Some("podman-machine-default".to_string())) +} + +fn machine_inspect_targets(connection: &str) -> Vec { + let mut names = vec![connection.to_string()]; + if let Some(machine) = connection.strip_suffix("-root") + && !machine.is_empty() + { + names.push(machine.to_string()); + } + names +} + +fn connection_list() -> Option { + let stdout = run_podman_capture(&["system", "connection", "list", "--format", "json"])?; + serde_json::from_slice(&stdout).ok() +} + +fn default_machine_connection(connections: Option<&serde_json::Value>) -> Option { + connections? + .as_array()? + .iter() + .find(|connection| { + connection["Default"].as_bool().unwrap_or(false) + && connection["IsMachine"].as_bool().unwrap_or(false) + }) + .and_then(|connection| connection["Name"].as_str()) + .map(str::to_string) +} + +fn connection_name_for_uri(connections: &serde_json::Value, uri: &str) -> Option { + connections + .as_array()? + .iter() + .find(|connection| { + connection["IsMachine"].as_bool().unwrap_or(false) + && connection["URI"].as_str() == Some(uri) + }) + .and_then(|connection| connection["Name"].as_str()) + .map(str::to_string) +} + +fn parse_info_socket(info: &serde_json::Value) -> Option { + let path = info["host"]["remoteSocket"]["path"].as_str()?; + unix_url_socket_path(path).or_else(|| (!path.is_empty()).then(|| PathBuf::from(path))) +} + +fn parse_machine_socket(machines: &serde_json::Value) -> Option { + let path = machines.as_array()?.first()?["ConnectionInfo"]["PodmanSocket"]["Path"].as_str()?; + (!path.is_empty()).then(|| PathBuf::from(path)) +} + +fn unix_url_socket_path(url: &str) -> Option { + let path = url.trim().strip_prefix("unix://")?; + (!path.is_empty()).then(|| PathBuf::from(path)) +} + +fn env_var_nonempty(key: &str) -> Option { + std::env::var(key) + .ok() + .filter(|value| !value.trim().is_empty()) +} + +fn run_podman_capture(args: &[&str]) -> Option> { + run_bounded_command("podman", args, DISCOVERY_TIMEOUT) +} + +/// Capture stdout without allowing a stalled Podman machine, SSH transport, or +/// descendant holding the stdout pipe open to block gateway startup forever. +fn run_bounded_command(program: &str, args: &[&str], timeout: Duration) -> Option> { + use std::io::Read as _; + use std::sync::mpsc; + + let mut command = Command::new(program); + command + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + set_new_process_group(&mut command); + let mut child = command.spawn().ok()?; + + let mut stdout = child.stdout.take()?; + let (sender, receiver) = mpsc::channel(); + std::thread::spawn(move || { + let mut output = Vec::new(); + let _ = stdout.read_to_end(&mut output); + let _ = sender.send(output); + }); + + let deadline = Instant::now() + timeout; + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(DISCOVERY_POLL_INTERVAL); + } + Ok(None) | Err(_) => break None, + } + }; + + let Some(status) = status else { + terminate_process_group(&mut child); + let _ = child.wait(); + return None; + }; + + match receiver.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + Ok(output) if status.success() => Some(output), + Ok(_) => None, + Err(_) => { + terminate_process_group(&mut child); + None + } + } +} + +#[cfg(unix)] +fn set_new_process_group(command: &mut Command) { + use std::os::unix::process::CommandExt as _; + command.process_group(0); +} + +#[cfg(not(unix))] +fn set_new_process_group(_command: &mut Command) {} + +#[cfg(unix)] +fn terminate_process_group(child: &mut std::process::Child) { + let pid = i32::try_from(child.id()).unwrap_or(i32::MAX); + let group = nix::unistd::Pid::from_raw(-pid); + let _ = nix::sys::signal::kill(group, nix::sys::signal::Signal::SIGKILL); + let _ = child.kill(); +} + +#[cfg(not(unix))] +fn terminate_process_group(child: &mut std::process::Child) { + let _ = child.kill(); +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn parse_info_socket_rejects_missing_or_empty_paths() { + assert_eq!( + parse_info_socket(&json!({"host": {"remoteSocket": {}}})), + None + ); + assert_eq!( + parse_info_socket(&json!({"host": {"remoteSocket": {"path": ""}}})), + None + ); + } + + #[test] + fn parse_machine_socket_rejects_missing_socket_or_machine() { + assert_eq!(parse_machine_socket(&json!([])), None); + assert_eq!(parse_machine_socket(&json!([{"ConnectionInfo": {}}])), None); + } + + #[test] + fn parses_native_and_machine_socket_paths() { + let info = json!({"host": {"remoteSocket": {"path": "unix:///run/user/1000/podman.sock"}}}); + assert_eq!( + parse_info_socket(&info), + Some(PathBuf::from("/run/user/1000/podman.sock")) + ); + + let machine = json!([{"ConnectionInfo": {"PodmanSocket": {"Path": "/tmp/machine.sock"}}}]); + assert_eq!( + parse_machine_socket(&machine), + Some(PathBuf::from("/tmp/machine.sock")) + ); + } + + #[test] + fn resolves_named_rootful_and_default_machine_connections() { + let connections = json!([ + {"Name": "team-machine-root", "URI": "ssh://team", "Default": true, "IsMachine": true}, + {"Name": "remote", "URI": "ssh://remote", "Default": false, "IsMachine": false} + ]); + assert_eq!( + active_machine(Some("team-machine-root"), None, Some(&connections)), + Some("team-machine-root".to_string()) + ); + assert_eq!( + machine_inspect_targets("team-machine-root"), + ["team-machine-root", "team-machine"] + ); + assert_eq!( + active_machine(None, None, Some(&connections)), + Some("team-machine-root".to_string()) + ); + } + + #[test] + fn explicit_non_machine_endpoint_does_not_guess_a_machine() { + let connections = json!([ + {"Name": "local", "URI": "ssh://local", "Default": true, "IsMachine": true}, + {"Name": "remote", "URI": "ssh://remote", "Default": false, "IsMachine": false} + ]); + assert_eq!( + active_machine(None, Some("ssh://remote"), Some(&connections)), + None + ); + } + + #[test] + fn container_host_maps_to_the_matching_machine_connection() { + let connections = json!([ + {"Name": "work", "URI": "ssh://core@127.0.0.1:5555/run/podman.sock", "Default": false, "IsMachine": true}, + {"Name": "podman-machine-default", "URI": "ssh://core@127.0.0.1:4444/run/podman.sock", "Default": true, "IsMachine": true} + ]); + assert_eq!( + active_machine( + None, + Some("ssh://core@127.0.0.1:5555/run/podman.sock"), + Some(&connections), + ), + Some("work".to_string()) + ); + } + + #[test] + fn unix_url_socket_path_only_accepts_nonempty_unix_urls() { + assert_eq!( + unix_url_socket_path("unix:///run/user/1000/podman.sock"), + Some(PathBuf::from("/run/user/1000/podman.sock")) + ); + assert_eq!(unix_url_socket_path("ssh://core@127.0.0.1/x"), None); + assert_eq!(unix_url_socket_path("tcp://127.0.0.1:2375"), None); + assert_eq!(unix_url_socket_path("unix://"), None); + } + + #[cfg(unix)] + #[test] + fn bounded_command_captures_stdout_on_success() { + assert_eq!( + run_bounded_command("printf", &["hello"], Duration::from_secs(5)), + Some(b"hello".to_vec()) + ); + } + + #[cfg(unix)] + #[test] + fn bounded_command_returns_none_on_nonzero_exit_or_missing_program() { + assert_eq!( + run_bounded_command("false", &[], Duration::from_secs(5)), + None + ); + assert_eq!( + run_bounded_command( + "openshell-nonexistent-binary-xyz", + &[], + Duration::from_secs(5), + ), + None + ); + } + + #[cfg(unix)] + #[test] + fn bounded_command_kills_child_that_exceeds_deadline() { + let start = Instant::now(); + let result = run_bounded_command("sleep", &["30"], Duration::from_millis(200)); + assert_eq!(result, None); + assert!( + start.elapsed() < Duration::from_secs(5), + "bounded command did not return promptly" + ); + } + + #[cfg(unix)] + #[test] + fn bounded_command_bounds_drain_when_in_group_descendant_holds_stdout() { + let start = Instant::now(); + let result = run_bounded_command( + "sh", + &["-c", "sleep 30 & echo done"], + Duration::from_millis(300), + ); + assert_eq!(result, None); + assert!( + start.elapsed() < Duration::from_secs(2), + "drain blocked on a descendant holding stdout" + ); + } + + #[cfg(unix)] + #[test] + fn bounded_command_bounds_drain_when_descendant_escapes_process_group() { + let start = Instant::now(); + let result = run_bounded_command( + "bash", + &["-c", "set -m; sleep 5 & echo done"], + Duration::from_millis(300), + ); + assert_eq!(result, None); + assert!( + start.elapsed() < Duration::from_secs(2), + "drain blocked on a descendant that escaped the process group" + ); + } +} diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index 30a0f2a970..0d250b6e71 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -22,6 +22,7 @@ path = "src/main.rs" openshell-core = { path = "../openshell-core", default-features = false } openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } +openshell-driver-podman = { path = "../openshell-driver-podman" } openshell-vfio = { path = "../openshell-vfio" } bollard = { version = "0.20", features = ["ssh"] } diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index bc11a1453c..129965188d 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -3679,28 +3679,7 @@ async fn connect_local_container_engine() -> Option { } fn detect_podman_socket() -> Option { - let mut candidates = Vec::new(); - if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") - && !path.trim().is_empty() - { - candidates.push(PathBuf::from(path)); - } - if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { - candidates.push(PathBuf::from(runtime_dir).join("podman/podman.sock")); - } - #[cfg(target_os = "linux")] - candidates.push(PathBuf::from(format!( - "/run/user/{}/podman/podman.sock", - rustix::process::geteuid().as_raw() - ))); - if let Some(home) = std::env::var_os("HOME") { - candidates - .push(PathBuf::from(home).join(".local/share/containers/podman/machine/podman.sock")); - } - openshell_core::local_api_socket::first_responsive_socket(&candidates, |response| { - openshell_core::local_api_socket::http_response_is_success(response) - && openshell_core::local_api_socket::contains_ascii(response, b"Libpod-Api-Version:") - }) + openshell_driver_podman::driver::detect_socket() } fn is_openshell_local_build_image_ref(image_ref: &str) -> bool { diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml index 3f393c850f..f9d02027e0 100644 --- a/crates/openshell-gateway/Cargo.toml +++ b/crates/openshell-gateway/Cargo.toml @@ -34,6 +34,9 @@ tonic = { workspace = true, optional = true } tower = { workspace = true, optional = true } tracing = { workspace = true, optional = true } +[target.'cfg(target_os = "windows")'.dependencies] +openshell-driver-mxc = { path = "../openshell-driver-mxc", optional = true } + [features] default = ["telemetry", "in-tree-compute-drivers"] in-tree-compute-drivers = [ @@ -48,6 +51,7 @@ in-tree-compute-drivers = [ "dep:tonic", "dep:tower", "dep:tracing", + "dep:openshell-driver-mxc", ] telemetry = ["openshell-core/telemetry", "openshell-server/telemetry"] ## Convenience alias: every default feature except `telemetry`. Build a diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 6a5b503b29..ee2beacd5f 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -19,9 +19,9 @@ compile_error!( #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] mod vm; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[cfg(feature = "in-tree-compute-drivers")] use openshell_core::telemetry::TelemetryComputeDriver; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[cfg(feature = "in-tree-compute-drivers")] use openshell_server::ComputeDriverRegistration; use openshell_server::ComputeDriverRegistry; @@ -32,7 +32,79 @@ pub fn install_default_compute_drivers() -> ComputeDriverRegistry { let mut registry = ComputeDriverRegistry::new(); #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] install_in_tree_compute_drivers(&mut registry); + #[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] + install_mxc_compute_driver(&mut registry); + registry +} + +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] +fn install_mxc_compute_driver(registry: &mut ComputeDriverRegistry) { + let registration = ComputeDriverRegistration::new("mxc", u16::MAX, None, MxcFactory) + .expect("first-party driver name is valid") + .with_telemetry_category(TelemetryComputeDriver::anonymous_category("mxc")) + .with_local_singleplayer(); registry + .install(registration) + .expect("first-party driver names are unique"); + + for name in ["docker", "kubernetes", "podman", "vm"] { + let registration = ComputeDriverRegistration::new( + name, + u16::MAX, + None, + UnsupportedWindowsFactory { name }, + ) + .expect("first-party driver name is valid"); + registry + .install(registration) + .expect("first-party driver names are unique"); + } +} + +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] +#[derive(Clone, Copy)] +struct UnsupportedWindowsFactory { + name: &'static str, +} + +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] +#[async_trait::async_trait] +impl openshell_server::ComputeDriverFactory for UnsupportedWindowsFactory { + async fn build( + &self, + _context: openshell_server::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + Err(unsupported_windows_compute_driver(self.name)) + } +} + +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] +fn unsupported_windows_compute_driver(name: &str) -> openshell_core::Error { + openshell_core::Error::config(format!("compute driver '{name}' is unsupported on Windows")) +} + +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] +#[derive(Clone, Copy)] +struct MxcFactory; + +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] +#[async_trait::async_trait] +impl openshell_server::ComputeDriverFactory for MxcFactory { + async fn build( + &self, + context: openshell_server::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + let config: openshell_driver_mxc::MxcComputeConfig = context.driver_config()?; + let backend = openshell_driver_mxc::MxcComputeBackend::new(config); + let sink = backend.policy_sink(); + let driver = openshell_driver_mxc::ComputeDriverService::new(backend); + Ok( + openshell_server::ComputeDriverInstance::InProcessWithSandboxPolicy { + driver: std::sync::Arc::new(driver), + sink, + }, + ) + } } #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] @@ -122,8 +194,10 @@ fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] fn kubernetes_tracing_setup( otlp_endpoint: Option<&str>, + gateway_name: Option<&str>, ) -> openshell_server::ComputeDriverTracingSetup { - let (provider, error) = openshell_driver_kubernetes::otel_tracing::provider_for(otlp_endpoint); + let (provider, error) = + openshell_driver_kubernetes::otel_tracing::provider_for(otlp_endpoint, gateway_name); let layer = provider.as_ref().map(|provider| { let layer: openshell_server::ComputeDriverTracingLayer = Box::new( openshell_driver_kubernetes::otel_tracing::in_process_layer(provider), @@ -146,8 +220,10 @@ fn kubernetes_tracing_setup( #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] fn podman_tracing_setup( otlp_endpoint: Option<&str>, + gateway_name: Option<&str>, ) -> openshell_server::ComputeDriverTracingSetup { - let (provider, error) = openshell_driver_podman::otel_tracing::provider_for(otlp_endpoint); + let (provider, error) = + openshell_driver_podman::otel_tracing::provider_for(otlp_endpoint, gateway_name); let layer = provider.as_ref().map(|provider| { let layer: openshell_server::ComputeDriverTracingLayer = Box::new( openshell_driver_podman::otel_tracing::in_process_layer(provider), @@ -170,8 +246,10 @@ fn podman_tracing_setup( #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] fn docker_tracing_setup( otlp_endpoint: Option<&str>, + gateway_name: Option<&str>, ) -> openshell_server::ComputeDriverTracingSetup { - let (provider, error) = openshell_driver_docker::otel_tracing::provider_for(otlp_endpoint); + let (provider, error) = + openshell_driver_docker::otel_tracing::provider_for(otlp_endpoint, gateway_name); let layer = provider.as_ref().map(|provider| { let layer: openshell_server::ComputeDriverTracingLayer = Box::new( openshell_driver_docker::otel_tracing::in_process_layer(provider), @@ -324,8 +402,13 @@ impl openshell_server::ComputeDriverFactory for VmFactory { &mut config.guest_tls_key, context.guest_tls_paths(), ); - let endpoint = - vm::spawn(context.gateway_log_level(), &config, context.otlp_config()).await?; + let endpoint = vm::spawn( + context.gateway_log_level(), + context.gateway_name(), + &config, + context.otlp_config(), + ) + .await?; Ok(openshell_server::ComputeDriverInstance::ManagedRemote( endpoint, )) @@ -349,3 +432,25 @@ fn apply_guest_tls( *key = Some(default_key.to_owned()); } } + +#[cfg(all(test, target_os = "windows", feature = "in-tree-compute-drivers"))] +mod windows_tests { + use super::*; + + #[test] + fn windows_builtin_compute_drivers_report_unsupported() { + let registry = install_default_compute_drivers(); + assert_eq!( + registry.installed_driver_names().collect::>(), + ["docker", "kubernetes", "mxc", "podman", "vm"] + ); + + for name in ["docker", "kubernetes", "podman", "vm"] { + let message = unsupported_windows_compute_driver(name).to_string(); + assert!( + message.contains("unsupported on Windows"), + "{name} rejection should be explicit, got: {message}" + ); + } + } +} diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index 32e0a14813..d06e6fbc8f 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -9,11 +9,7 @@ use crate::config_file; use crate::defaults::LocalTlsPaths; -#[cfg(target_os = "windows")] -use openshell_core::ComputeDriverKind; use openshell_core::{Error, Result}; -#[cfg(target_os = "windows")] -use openshell_driver_mxc::MxcComputeConfig; use serde::Deserialize; use std::collections::BTreeMap; use std::path::PathBuf; @@ -50,15 +46,6 @@ pub struct DriverStartupContext<'a> { pub endpoint_overrides: &'a BTreeMap, } -/// Build the selected MXC config from TOML. MXC is Windows-only and has no -/// runtime-default overlay; the driver reads its own settings from the config. -/// The Linux built-in driver configs now live in the `builtin` submodule -/// (compiled only off Windows). -#[cfg(target_os = "windows")] -pub fn mxc_config_from_context(context: DriverStartupContext<'_>) -> Result { - driver_config_from_context(context, ComputeDriverKind::Mxc.as_str()) -} - pub fn remote_driver_config_from_context( context: DriverStartupContext<'_>, name: &str, diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index b8f13272fe..fa3e0f2c11 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -564,14 +564,10 @@ pub struct ComputeRuntime { lifecycle_gates: Arc, gateway_listener_requirements: Vec, replica_id: String, - /// A1 policy side channel for the in-process MXC driver. `create_sandbox` - /// stages the typed `SandboxPolicy` here by sandbox id immediately before - /// dispatching to the driver, which consumes it. `None` for all other - /// drivers. The proto driver contract has no `policy` field and there is no - /// driver-side `GetSandboxConfig`, so this in-process map is how the policy - /// reaches the MXC backend without changing the cross-process contract. + /// Optional policy side channel supplied by an in-process driver whose + /// public RPC contract cannot carry the typed sandbox policy. #[cfg(target_os = "windows")] - mxc_policy_sink: Option>>>, + sandbox_policy_sink: Option>>>, } impl fmt::Debug for ComputeRuntime { @@ -691,7 +687,7 @@ impl ComputeRuntime { gateway_listener_requirements, replica_id: lease::replica_id(), #[cfg(target_os = "windows")] - mxc_policy_sink: None, + sandbox_policy_sink: None, }) } @@ -793,6 +789,16 @@ impl ComputeRuntime { self } + #[cfg(target_os = "windows")] + #[must_use] + pub(crate) fn with_sandbox_policy_sink( + mut self, + sink: Arc>>, + ) -> Self { + self.sandbox_policy_sink = Some(sink); + self + } + #[must_use] pub(crate) fn gateway_listener_requirements(&self) -> &[GatewayListenerRequirement] { &self.gateway_listener_requirements @@ -912,7 +918,7 @@ impl ComputeRuntime { // before dispatch. The driver removes/consumes it in create_sandbox. The // proto driver contract has no policy field, so this is the only path. #[cfg(target_os = "windows")] - if let Some(sink) = &self.mxc_policy_sink + if let Some(sink) = &self.sandbox_policy_sink && let Some(p) = sandbox.spec.as_ref().and_then(|s| s.policy.clone()) { sink.lock().await.insert(sandbox_id.clone(), p); @@ -4310,10 +4316,24 @@ fn derive_phase(status: Option<&DriverSandboxStatus>) -> SandboxPhase { return SandboxPhase::Deleting; } - if status.conditions.iter().any(|condition| { - condition.r#type.eq_ignore_ascii_case("Suspended") + // `Ready=True` means the sandbox is usable through this gateway and must + // win over a `Suspended=True` condition. Agent Sandbox v1beta1 sets + // `Suspended=True (PodTerminated)` on stop and does not clear it on resume, + // so a resumed CR carries both `Ready=True` and a stale `Suspended=True`. + // Treating any `Suspended=True` as Stopped would pin the resumed sandbox at + // Starting forever (issue #2932). A genuine stop leaves `Ready` unset or + // False, so `Suspended` still resolves to Stopped in that case. + let ready = status.conditions.iter().any(|condition| { + condition.r#type.eq_ignore_ascii_case("Ready") && condition.status.eq_ignore_ascii_case("true") - }) { + }); + + if !ready + && status.conditions.iter().any(|condition| { + condition.r#type.eq_ignore_ascii_case("Suspended") + && condition.status.eq_ignore_ascii_case("true") + }) + { return SandboxPhase::Stopped; } @@ -4641,7 +4661,7 @@ pub async fn new_test_runtime_with_driver( gateway_listener_requirements: Vec::new(), replica_id: "test-replica".to_string(), #[cfg(target_os = "windows")] - mxc_policy_sink: None, + sandbox_policy_sink: None, } } @@ -5369,7 +5389,7 @@ mod tests { gateway_listener_requirements: Vec::new(), replica_id: "test-replica".to_string(), #[cfg(target_os = "windows")] - mxc_policy_sink: None, + sandbox_policy_sink: None, } } @@ -7217,6 +7237,56 @@ mod tests { assert_eq!(current.phase(), SandboxPhase::Stopped as i32); } + #[tokio::test] + async fn resumed_v1beta1_snapshot_with_stale_suspended_reaches_ready() { + // Reproduces issue #2932: on Agent Sandbox v1beta1 a resumed CR reports + // Ready=True (DependenciesReady) alongside a stale Suspended=True + // (PodTerminated). Starting from the Starting phase that `start` sets, the + // reconciled sandbox must advance to Ready rather than being pinned at + // Starting by the stale Suspended condition. + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let sandbox = sandbox_record("sb-resumed", "sandbox-resumed", SandboxPhase::Starting); + runtime.store.put_message(&sandbox).await.unwrap(); + register_test_supervisor_session(&runtime, sandbox.object_id()); + + let mut resumed = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name()); + resumed.status = Some(DriverSandboxStatus { + sandbox_name: sandbox.object_name().to_string(), + instance_id: format!("{}-pod", sandbox.object_name()), + conditions: vec![ + DriverCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: "DependenciesReady".to_string(), + message: "Sandbox is ready".to_string(), + last_transition_time: String::new(), + }, + DriverCondition { + r#type: "Suspended".to_string(), + status: "True".to_string(), + reason: "PodTerminated".to_string(), + message: "Pod terminated".to_string(), + last_transition_time: String::new(), + }, + ], + ..Default::default() + }); + + runtime.apply_sandbox_update(resumed).await.unwrap(); + + let current = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!( + current.phase(), + SandboxPhase::Ready as i32, + "a resumed, Ready sandbox must not stay Starting because of a stale Suspended condition" + ); + } + #[tokio::test] async fn stopped_container_snapshot_cannot_error_stopped_sandbox() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 83bdc0b00b..abb59fadde 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -3136,12 +3136,11 @@ pub(super) async fn handle_get_sandbox_provider_environment( // --------------------------------------------------------------------------- fn validate_live_policy_update_support( - driver_kind: Option, + driver_name: &str, has_policy: bool, has_merge_ops: bool, ) -> Result<(), Status> { - if (has_policy || has_merge_ops) && driver_kind == Some(openshell_core::ComputeDriverKind::Mxc) - { + if (has_policy || has_merge_ops) && driver_name == "mxc" { return Err(Status::failed_precondition( "live policy updates are not supported for MXC sandboxes; recreate the sandbox so the new policy is mapped before launch", )); @@ -3223,7 +3222,11 @@ async fn handle_update_config_inner( "one of policy, setting_key, or merge_operations must be provided", )); } - validate_live_policy_update_support(state.compute.driver_kind(), has_policy, has_merge_ops)?; + validate_live_policy_update_support( + state.compute.configured_driver_name(), + has_policy, + has_merge_ops, + )?; if req.global { if !req.annotations.is_empty() { return Err(Status::invalid_argument( @@ -6914,21 +6917,14 @@ mod tests { #[test] fn mxc_rejects_sandbox_policy_replacement_and_merge_updates() { for (has_policy, has_merge_ops) in [(true, false), (false, true)] { - let error = validate_live_policy_update_support( - Some(openshell_core::ComputeDriverKind::Mxc), - has_policy, - has_merge_ops, - ) - .expect_err("MXC must reject policy mutations after launch"); + let error = validate_live_policy_update_support("mxc", has_policy, has_merge_ops) + .expect_err("MXC must reject policy mutations after launch"); assert_eq!(error.code(), Code::FailedPrecondition); } - let error = validate_live_policy_update_support( - Some(openshell_core::ComputeDriverKind::Mxc), - true, - false, - ) - .expect_err("global policy replacement also changes desired state for live MXC sandboxes"); + let error = validate_live_policy_update_support("mxc", true, false).expect_err( + "global policy replacement also changes desired state for live MXC sandboxes", + ); assert_eq!(error.code(), Code::FailedPrecondition); } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index ba56376ac8..0b888b3609 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1051,6 +1051,12 @@ pub use compute::{ pub enum ComputeDriverInstance { /// A driver hosted in the gateway process. InProcess(SharedComputeDriver), + /// An in-process driver with a typed sandbox-policy side channel. + #[cfg(target_os = "windows")] + InProcessWithSandboxPolicy { + driver: SharedComputeDriver, + sink: Arc>>, + }, /// A driver process launched and owned by the gateway. ManagedRemote(AcquiredRemoteDriverEndpoint), } @@ -1090,7 +1096,7 @@ impl ComputeDriverTracingSetup { } /// Factory for a compiled driver's optional tracing integration. -pub type ComputeDriverTracingFactory = fn(Option<&str>) -> ComputeDriverTracingSetup; +pub type ComputeDriverTracingFactory = fn(Option<&str>, Option<&str>) -> ComputeDriverTracingSetup; /// Factory for a compute driver linked into a gateway binary. #[async_trait::async_trait] @@ -1264,6 +1270,7 @@ impl ComputeDriverRegistry { selection: &ComputeDriverSelection, endpoint_overrides: &BTreeMap, otlp_endpoint: Option<&str>, + gateway_name: Option<&str>, ) -> ComputeDriverTracingSetup { let name = selection.name(); if endpoint_overrides.contains_key(name) { @@ -1272,7 +1279,7 @@ impl ComputeDriverRegistry { self.get(name) .and_then(|registration| registration.tracing_setup) .map_or_else(ComputeDriverTracingSetup::default, |setup| { - setup(otlp_endpoint) + setup(otlp_endpoint, gateway_name) }) } @@ -1322,6 +1329,7 @@ impl ComputeDriverRegistry { pub struct ComputeDriverBuildContext<'a> { driver_name: String, + gateway_name: &'a str, gateway_bind_address: SocketAddr, gateway_log_level: &'a str, driver_startup: compute::driver_config::DriverStartupContext<'a>, @@ -1335,6 +1343,11 @@ impl ComputeDriverBuildContext<'_> { &self.driver_name } + #[must_use] + pub fn gateway_name(&self) -> &str { + self.gateway_name + } + #[must_use] pub fn gateway_bind_address(&self) -> SocketAddr { self.gateway_bind_address @@ -1419,6 +1432,7 @@ async fn build_compute_runtime( ConfiguredComputeDriver::Registered(registration) => { let build_context = ComputeDriverBuildContext { driver_name: registration.name.clone(), + gateway_name: &config.name, gateway_bind_address: config.bind_address, gateway_log_level: &config.log_level, driver_startup, @@ -1441,6 +1455,24 @@ async fn build_compute_runtime( .map_err(|error| { Error::execution(format!("failed to create compute runtime: {error}")) })?, + #[cfg(target_os = "windows")] + ComputeDriverInstance::InProcessWithSandboxPolicy { driver, sink } => { + ComputeRuntime::from_driver( + registration.name, + driver, + None, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions, + ) + .await + .map_err(|error| { + Error::execution(format!("failed to create compute runtime: {error}")) + })? + .with_sandbox_policy_sink(sink) + } ComputeDriverInstance::ManagedRemote(mut endpoint) => { endpoint.name = registration.name; ComputeRuntime::new_remote_driver( diff --git a/crates/openshell-server/src/otel_tracing.rs b/crates/openshell-server/src/otel_tracing.rs index fc9afe3d18..16a94a7321 100644 --- a/crates/openshell-server/src/otel_tracing.rs +++ b/crates/openshell-server/src/otel_tracing.rs @@ -51,11 +51,6 @@ impl<'a> GatewayResourceAttributes<'a> { compute_driver, } } - - /// The configured gateway installation name, if any. - pub fn name(&self) -> Option<&'a str> { - self.name - } } fn trace_config<'cfg>( diff --git a/crates/openshell-server/src/tracing_setup.rs b/crates/openshell-server/src/tracing_setup.rs index 5f9eac9a84..2bc583dac7 100644 --- a/crates/openshell-server/src/tracing_setup.rs +++ b/crates/openshell-server/src/tracing_setup.rs @@ -12,6 +12,7 @@ use tracing_subscriber::EnvFilter; use tracing_subscriber::prelude::*; use crate::config_file::OtlpConfig; +use crate::otel_tracing::GatewayResourceAttributes; use crate::tracing_bus::TracingLogBus; use crate::{ComputeDriverTracingSetup, ComputeDriverTracingShutdown}; @@ -40,8 +41,9 @@ pub fn install( tracing_log_bus: &TracingLogBus, otlp_config: Option<&OtlpConfig>, compute_driver_tracing: ComputeDriverTracingSetup, + gateway: GatewayResourceAttributes<'_>, ) -> (TracingHandle, Option) { - let (tracer_provider, setup_error) = crate::otel_tracing::provider_for(otlp_config); + let (tracer_provider, setup_error) = crate::otel_tracing::provider_for(otlp_config, gateway); let ComputeDriverTracingSetup { layer, shutdown, diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 82aa68a956..ffa18f1b34 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -46,7 +46,7 @@ Reserved built-in values are `docker`, `podman`, `kubernetes`, and `vm`. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. -When `compute_drivers` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Local container runtimes must respond to an API probe before the gateway selects them. The VM driver is never auto-detected; configure it explicitly with `compute_drivers = ["vm"]` or set `OPENSHELL_DRIVERS=vm` in the launch environment. +When `compute_drivers` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker. Docker must respond on a known API socket. Podman first probes known API sockets and then asks the `podman` CLI for the active native or machine-backed socket. The VM driver is never auto-detected; configure it explicitly with `compute_drivers = ["vm"]` or set `OPENSHELL_DRIVERS=vm` in the launch environment. Common gateway options: @@ -227,7 +227,7 @@ namespace roots. These checks do not make host bind mounts safe. [Podman](https://podman.io/)-backed sandboxes run as rootless containers on the gateway host. Use Podman for Linux workstation workflows that avoid a rootful Docker daemon. -The gateway talks to the Podman API socket. The Podman driver requires Podman 5.x, cgroups v2, rootless networking, and an active Podman user socket. When `socket_path` is not set, the driver probes for a responsive Podman socket and fails to start if none respond. +The gateway talks to the Podman API socket. The Podman driver requires Podman 5.x, cgroups v2, rootless networking, and an active Podman user socket. When `socket_path` is not set, the driver probes known socket paths, then uses the `podman` CLI to resolve the active native or machine-backed connection. It fails to start if neither method finds a socket. For maintainer-level implementation details, refer to the [Podman driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/README.md) and [Podman networking notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-podman/NETWORKING.md). @@ -340,7 +340,7 @@ The gateway starts `openshell-driver-vm` over a private Unix socket and passes i ### Local image resolution -The VM driver resolves sandbox images from a local container engine before falling back to registry pulls. It tries Docker first, then falls back to the Podman socket (Docker-compatible API). On Linux with Podman, enable the API socket so the driver can find local images: +The VM driver resolves sandbox images from a local container engine before falling back to registry pulls. It tries Docker first, then uses the same Podman socket discovery as the Podman driver. On Linux with Podman, enable the API socket so the driver can find local images: ```shell systemctl --user start podman.socket diff --git a/tasks/scripts/windows-msvc.ps1 b/tasks/scripts/windows-msvc.ps1 index cf1b4c8686..0a0410f4ca 100644 --- a/tasks/scripts/windows-msvc.ps1 +++ b/tasks/scripts/windows-msvc.ps1 @@ -583,7 +583,7 @@ function Assert-GatewayExcludesUnsupportedDriverCrates([string] $RustTarget) { $logName = "build-$RustTarget-driver-tree.log" Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo tree -p openshell-server --target $RustTarget --prefix none" ` + -CargoArgs "cargo tree -p openshell-gateway --target $RustTarget --prefix none" ` -LogName $logName $logPath = Join-Path $LogDir $logName @@ -643,7 +643,7 @@ function Invoke-UnsupportedContractTests([string] $RustTarget) { foreach ($test in $tests) { Invoke-VsCargo ` -RustTarget $RustTarget ` - -CargoArgs "cargo test -p openshell-server --target $RustTarget $test $Z3ServerFeatures" ` + -CargoArgs "cargo test -p openshell-gateway --target $RustTarget $test $Z3ServerFeatures" ` -LogName "test-$RustTarget-unsupported-$test.log" } } From 5c5b7a8e7173c42ff196d0c0529c31d2a3122947 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 31 Aug 2026 21:31:40 -0700 Subject: [PATCH 8/9] refactor(compute): remove MXC policy side channel Signed-off-by: Drew Newberry --- architecture/README.md | 9 +- architecture/compute-runtimes.md | 57 ++++--- architecture/windows-msvc-build.md | 3 + crates/openshell-driver-docker/src/lib.rs | 2 + crates/openshell-driver-docker/src/tests.rs | 10 +- .../openshell-driver-kubernetes/src/driver.rs | 2 + .../openshell-driver-kubernetes/src/grpc.rs | 5 +- crates/openshell-driver-mxc/README.md | 13 +- crates/openshell-driver-mxc/src/driver.rs | 123 +++++++------- crates/openshell-driver-mxc/src/grpc.rs | 11 +- crates/openshell-driver-mxc/src/policy.rs | 4 +- crates/openshell-driver-podman/src/driver.rs | 2 + crates/openshell-driver-podman/src/grpc.rs | 1 + crates/openshell-driver-vm/src/driver.rs | 5 + crates/openshell-gateway/src/lib.rs | 10 +- crates/openshell-server/src/compute/mod.rs | 158 +++++++++++++----- crates/openshell-server/src/grpc/policy.rs | 27 +-- crates/openshell-server/src/lib.rs | 24 --- crates/openshell-server/src/test_support.rs | 22 ++- docs/reference/sandbox-compute-drivers.mdx | 15 +- proto/compute_driver.proto | 20 +++ 21 files changed, 329 insertions(+), 194 deletions(-) diff --git a/architecture/README.md b/architecture/README.md index 0d814a7a22..5b77af1f8d 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -141,9 +141,12 @@ bridge networks, port mappings, NAT traversal, or bespoke tunnels. The common runtime requirement is narrower: the supervisor must be able to reach the gateway. -The Windows MXC driver is an explicit exception. It launches and monitors a -one-shot workload in the driver, self-reports readiness, and does not expose a -supervisor session, interactive connect, live policy delivery, or governed egress. +The compute-driver capability contract identifies whether the supervisor or the +driver owns runtime control. Supervisor-controlled drivers use the session model +above. A driver-controlled runtime receives the canonical policy in its validate +and create requests, applies it before launch, and self-reports readiness without +a supervisor session. The Windows MXC driver uses this mode and does not expose +interactive connect, live policy delivery, or governed egress. The gateway delivers desired state; the sandbox applies it locally. Policy, settings, credentials, and inference routes flow from the gateway to the diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index ff28d9899b..89584ebc43 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -1,34 +1,42 @@ # Compute Runtimes Compute runtimes create, stop, start, delete, and watch sandbox workloads for the -gateway. They do not replace sandbox policy enforcement. Every runtime starts a -workload that runs the `openshell-sandbox` supervisor, and the supervisor -enforces the sandbox contract locally. +gateway. Supervisor-controlled runtimes start a workload that runs the +`openshell-sandbox` supervisor, which enforces the sandbox contract locally. +Driver-controlled runtimes apply the canonical sandbox policy while +provisioning and report workload readiness directly. ## Driver Contract -Each runtime receives a sandbox spec from the gateway and is responsible for: +Each runtime receives a sandbox spec and canonical policy from the gateway and +is responsible for: - Selecting the sandbox image. -- Injecting sandbox identity and gateway callback configuration. -- Supplying TLS or secret material for supervisor callbacks. -- Providing the supervisor binary or image in the workload. +- For supervisor-controlled runtimes, injecting sandbox identity and gateway + callback configuration, supplying callback credentials, and providing the + supervisor binary or image. +- For driver-controlled runtimes, validating and applying the canonical policy + before launching the workload. - Forwarding the exact canonical main-process argv and TTY mode without shell reconstruction. The sandbox-level environment and policy workspace apply to the main process. - Reporting lifecycle and platform events back to the gateway. - Cleaning up runtime-owned resources. -Drivers report **backend state only**. A driver snapshot with `Ready=True` means -the underlying compute resource (container, pod, VM) is healthy and running — -nothing more. Drivers must not gate on supervisor session state or hold -references to gateway-internal types. The gateway owns the public -`SandboxPhase::Ready` decision. This applies equally to extension drivers -implementing `ComputeDriver` out of tree. +Drivers report **runtime-observed state only** and must not hold references to +gateway-internal types. For supervisor-controlled runtimes, `Ready=True` means +only that the compute resource is healthy; the gateway also requires a +supervisor session before publishing `SandboxPhase::Ready`. For +driver-controlled runtimes, `Ready=True` is authoritative because the driver +launches and monitors the policy-constrained workload itself. `compute_driver.proto` is the supported gateway/driver extension boundary. At initialization the gateway snapshots the driver's identity, version, -default image, and gateway-lifecycle preference from `GetCapabilities`. +default image, gateway-lifecycle preference, and `sandbox_runtime_control` from +`GetCapabilities`. An omitted or unknown runtime-control value is treated as +`SUPERVISOR` for compatibility. The gateway includes the canonical +`SandboxPolicy` in both `ValidateSandboxCreate` and `CreateSandbox`; drivers +that advertise `DRIVER` must validate and apply it before launch. Process-identity omissions are preserved across this boundary so every driver can apply its native image or runtime defaults. Driver-requested listeners are structurally validated and remain restricted to sandbox callback RPCs. @@ -45,28 +53,29 @@ reason strings. ## Sandbox Readiness Composition -The gateway composes driver backend state with supervisor session presence to -produce the public `SandboxPhase`. This composition is gateway-owned and applied -uniformly across all drivers: +The gateway composes driver state with the advertised runtime-control mode to +produce the public `SandboxPhase`: ``` backend_phase = derive_phase(driver_status) public_phase = if backend_phase in {Error, Deleting}: → pass through (terminal precedence) + if runtime_control == Driver && backend_phase == Ready: → Ready if backend_phase == Ready && session connected: → Ready if backend_phase == Ready && no session: → Provisioning if backend_phase in {Provisioning, Unknown} && session: → Ready if backend_phase in {Provisioning, Unknown} && no session: → Provisioning ``` -When `public_phase == Ready` the sandbox is usable through the gateway — both the +For a supervisor-controlled runtime, `public_phase == Ready` means both the backend resource is healthy and a supervisor session is registered. A sandbox whose backend reports ready but has no supervisor session yet holds `Provisioning` with a `Ready=False`, `SupervisorNotConnected` condition and the message `Backend ready; waiting for supervisor session`. This distinguishes it from a sandbox whose compute resource is still provisioning without exposing contradictory public -readiness signals. +readiness signals. For a driver-controlled runtime, the driver's ready condition +is published without waiting for a supervisor session. **Session precedence over lagging driver snapshots:** A supervisor session can only be established by a running workload. When `set_supervisor_session_state` promotes the @@ -85,10 +94,12 @@ session-owning replica. That work is deferred to GitHub issue #1868. Until then, deployments that require reliable readiness composition must run a single gateway replica. -**Extension point:** The readiness decision is a safety invariant, not an -operator-configurable hook. The driver contract is the correct extension point for -custom backend readiness semantics. RFC-0010 lifecycle hooks may observe readiness -transitions via `post_commit`; they do not override the composition rule. +**Extension point:** Runtime control is a driver capability, not an +operator-configurable hook. A driver may advertise `DRIVER` only when it applies +the canonical create-time policy and owns workload readiness. Driver-controlled +runtimes do not support live policy mutation; recreate the sandbox to apply a +new policy. RFC-0010 lifecycle hooks may observe readiness transitions via +`post_commit`; they do not override the composition rule. The capability RPC reports driver identity, version, and the default sandbox image used by the gateway. GPU availability stays driver-local and is validated diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index 56927f81f4..8ee6c0e444 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -10,6 +10,9 @@ driver. It does not make Windows a Docker, Kubernetes, Podman, or VM runtime hos - Keep the Linux and macOS build paths unchanged. - Preserve gateway configuration parsing for all existing compute driver names. - Build and test the in-process MXC driver on supported Windows hosts. +- Use the ordinary in-process compute-driver composition path; MXC receives the + canonical sandbox policy through validate/create RPC messages and advertises + driver-owned runtime control for readiness. - Return clear unsupported errors when a Windows gateway is configured to use Docker, Kubernetes, Podman, or VM. - Keep dedicated `windows:*` validation tasks while allowing the repository-wide `pre-commit` task to delegate compiler-bearing Rust checks to the native diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d218593cf3..147ccc7aec 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -667,6 +667,8 @@ impl DockerComputeDriver { default_image: self.config.default_image.clone(), gateway_manages_lifecycle: true, supports_sandbox_authentication: false, + sandbox_runtime_control: + openshell_core::proto::compute::v1::SandboxRuntimeControl::Supervisor.into(), } } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index fd81c9b638..c8ab3f8082 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -222,7 +222,10 @@ async fn tracing_in_process_service_preserves_the_driver_rpc_server_boundary() { ); ComputeDriver::validate_sandbox_create( &service, - Request::new(ValidateSandboxCreateRequest { sandbox: None }), + Request::new(ValidateSandboxCreateRequest { + sandbox: None, + policy: None, + }), ) .instrument(gateway_span) .await @@ -306,7 +309,10 @@ async fn tracing_lifecycle_rpc_failures_export_docker_operation_spans() { async { ComputeDriver::create_sandbox( &driver, - Request::new(CreateSandboxRequest { sandbox: None }), + Request::new(CreateSandboxRequest { + sandbox: None, + policy: None, + }), ) .await .expect_err("missing sandbox should fail"); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index fdedc186d8..c858f103ed 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -575,6 +575,8 @@ impl KubernetesComputeDriver { default_image: self.config.default_image.clone(), gateway_manages_lifecycle: false, supports_sandbox_authentication: true, + sandbox_runtime_control: + openshell_core::proto::compute::v1::SandboxRuntimeControl::Supervisor.into(), }) } diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index 027c350a26..dbf609cd12 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -438,7 +438,10 @@ mod tests { ComputeDriver::validate_sandbox_create( &service, - Request::new(ValidateSandboxCreateRequest { sandbox: None }), + Request::new(ValidateSandboxCreateRequest { + sandbox: None, + policy: None, + }), ) .await } diff --git a/crates/openshell-driver-mxc/README.md b/crates/openshell-driver-mxc/README.md index 6496dfe286..7136e89a26 100644 --- a/crates/openshell-driver-mxc/README.md +++ b/crates/openshell-driver-mxc/README.md @@ -4,8 +4,11 @@ OpenShell compute driver backed by **Microsoft MXC** (`wxc-exec`) on Windows. ## Design -This driver implements the gateway's `ComputeDriver` contract as an in-process -library linked into `openshell-gateway`. `process_container` launches a one-shot +This driver implements the gateway's ordinary in-process `ComputeDriver` +contract and is linked into `openshell-gateway`. It advertises driver-owned +runtime control, so the gateway passes the canonical `SandboxPolicy` in both +`ValidateSandboxCreate` and `CreateSandbox` and accepts driver-reported +readiness without a supervisor session. `process_container` launches a one-shot AppContainer and is the default. The opt-in `isolation_session` backend uses the state-aware `provision` → `start` → `exec` → `stop` → `deprovision` lifecycle. The driver launches and monitors the configured workload itself and self-reports @@ -64,7 +67,11 @@ no isolation session needed), set `OPENSHELL_MXC_MOCK_WXC=1`. ## Policy mapping -The production driver maps the typed `SandboxPolicy` to MXC configuration before it inserts a registry entry or invokes `wxc-exec`. Mapping failure therefore returns from `CreateSandbox` without leaving a partial sandbox. +The production driver maps the typed `SandboxPolicy` carried by the standard +driver request to MXC configuration before it inserts a registry entry or +invokes `wxc-exec`. Mapping failure therefore returns from `CreateSandbox` +without leaving a partial sandbox. There is no in-process policy side channel +or MXC-specific gateway composition variant. `EmbeddedPolicyMapper` calls the embedded [`policy_map`](src/policy_map/) module directly and normalizes filesystem paths to Windows form. It does not add gateway-configured host paths. The policy supplied for the sandbox is the only source of filesystem grants. diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index d8e981a46d..edd7bff3a0 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -11,8 +11,9 @@ use openshell_core::gpu::{driver_gpu_requirements, effective_driver_gpu_count}; use openshell_core::proto::SandboxPolicy; use openshell_core::proto::compute::v1::{ DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, - GetCapabilitiesResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, - WatchSandboxesPlatformEvent, WatchSandboxesSandboxEvent, watch_sandboxes_event, + GetCapabilitiesResponse, SandboxRuntimeControl, WatchSandboxesDeletedEvent, + WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesSandboxEvent, + watch_sandboxes_event, }; use openshell_core::proto_struct::struct_to_json_value; use serde::{Deserialize, Serialize}; @@ -177,13 +178,6 @@ pub struct MxcComputeBackend { registry: Arc>>, watch_tx: Arc>, policy_mapper: Arc, - /// Out-of-band side channel for the `SandboxPolicy` (A1). The proto driver - /// contract has no `policy` field and there is no driver-side - /// `GetSandboxConfig`, so `ComputeRuntime::create_sandbox` stages the policy - /// here keyed by sandbox id (mirroring the `sandbox_token` injection), - /// immediately before dispatching to this backend's `create_sandbox`, which - /// removes/consumes it. - pending_policies: Arc>>, } impl std::fmt::Debug for MxcComputeBackend { @@ -279,17 +273,9 @@ impl MxcComputeBackend { // Production policy translation is always handled by the embedded // mapper before any MXC lifecycle side effects begin. policy_mapper: Arc::new(EmbeddedPolicyMapper), - pending_policies: Arc::new(Mutex::new(HashMap::new())), } } - /// Returns a clone of the `pending_policies` side channel so the gateway's - /// `ComputeRuntime` can stage the typed `SandboxPolicy` by sandbox id right - /// before dispatching `create_sandbox` (A1 wiring). - pub fn policy_sink(&self) -> Arc>> { - self.pending_policies.clone() - } - /// Test-only constructor wiring the in-process mock `wxc-exec` shim. #[cfg(test)] pub(crate) fn new_mocked(config: MxcComputeConfig) -> Self { @@ -304,10 +290,12 @@ impl MxcComputeBackend { driver_version: DRIVER_VERSION.to_string(), default_image: DEFAULT_IMAGE_SENTINEL.to_string(), gateway_manages_lifecycle: false, + supports_sandbox_authentication: false, + sandbox_runtime_control: SandboxRuntimeControl::Driver.into(), } } - pub fn validate_sandbox_create(&self, sandbox: &DriverSandbox) -> Result<(), tonic::Status> { + fn validate_sandbox_fields(&self, sandbox: &DriverSandbox) -> Result<(), tonic::Status> { if let Some(spec) = &sandbox.spec { if effective_driver_gpu_count(driver_gpu_requirements( spec.resource_requirements.as_ref(), @@ -330,6 +318,32 @@ impl MxcComputeBackend { sandbox_config(sandbox)?; Ok(()) } + + fn map_sandbox_policy( + &self, + sandbox_id: &str, + policy: Option<&SandboxPolicy>, + ) -> Result { + self.policy_mapper + .map( + policy, + &MapCtx { + sandbox_id: sandbox_id.to_string(), + egress: None, + }, + ) + .map_err(|error| tonic::Status::invalid_argument(error.to_string())) + } + + pub fn validate_sandbox_create( + &self, + sandbox: &DriverSandbox, + policy: Option<&SandboxPolicy>, + ) -> Result<(), tonic::Status> { + self.validate_sandbox_fields(sandbox)?; + self.map_sandbox_policy(&sandbox.id, policy)?; + Ok(()) + } pub async fn get_sandbox(&self, sandbox_name: &str) -> Option { let registry = self.registry.lock().await; registry @@ -343,28 +357,20 @@ impl MxcComputeBackend { registry.values().map(|e| e.sandbox.clone()).collect() } - pub async fn create_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), tonic::Status> { + pub async fn create_sandbox( + &self, + sandbox: &DriverSandbox, + policy: Option<&SandboxPolicy>, + ) -> Result<(), tonic::Status> { let sandbox_id = sandbox.id.clone(); - // Consume the out-of-band policy staged by `ComputeRuntime::create_sandbox` - // (A1). Always remove so rejected creates cannot leak policy state. - let policy = self.pending_policies.lock().await.remove(&sandbox_id); - self.validate_sandbox_create(sandbox)?; + self.validate_sandbox_fields(sandbox)?; let sandbox_config = sandbox_config(sandbox)?; // Policy translation is deterministic and side-effect free. Do it before // inserting the registry entry or launching MXC so invalid requests fail // synchronously at the CreateSandbox boundary. - let mapped = self - .policy_mapper - .map( - policy.as_ref(), - &MapCtx { - sandbox_id: sandbox_id.clone(), - egress: None, - }, - ) - .map_err(|error| tonic::Status::invalid_argument(error.to_string()))?; + let mapped = self.map_sandbox_policy(&sandbox_id, policy)?; if sandbox .spec @@ -1027,14 +1033,12 @@ mod lifecycle_tests { ]; let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); - // Stage the policy via the A1 side channel (as ComputeRuntime would). - let sink = backend.policy_sink(); - sink.lock() - .await - .insert("sb-pos".into(), fs_policy(&[&share])); - + let policy = fs_policy(&[&share]); let sb = driver_sandbox_with_command("sb-pos", &share, cmd); - backend.create_sandbox(&sb).await.expect("create accepted"); + backend + .create_sandbox(&sb, Some(&policy)) + .await + .expect("create accepted"); // Self-reported Ready=True (no supervisor) once the agent exec launches. let ready = wait_for(&backend, "sb-pos", |s| { @@ -1085,13 +1089,12 @@ mod lifecycle_tests { ]; let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); - let sink = backend.policy_sink(); - sink.lock() - .await - .insert("sb-pc".into(), fs_policy(&[&share])); - + let policy = fs_policy(&[&share]); let sb = driver_sandbox_with_command("sb-pc", &share, cmd); - backend.create_sandbox(&sb).await.expect("create accepted"); + backend + .create_sandbox(&sb, Some(&policy)) + .await + .expect("create accepted"); let ready = wait_for(&backend, "sb-pc", |s| { ready_condition(s).is_some_and(|c| c.status == "True" && c.reason == "AgentRunning") @@ -1142,12 +1145,12 @@ mod lifecycle_tests { // Subscribe to the watch stream BEFORE create so we catch the denial event. let mut stream = backend.watch_sandboxes().await; - let sink = backend.policy_sink(); - sink.lock() - .await - .insert("sb-neg".into(), fs_policy(&[&share])); + let policy = fs_policy(&[&share]); backend - .create_sandbox(&driver_sandbox_with_command("sb-neg", &share, cmd)) + .create_sandbox( + &driver_sandbox_with_command("sb-neg", &share, cmd), + Some(&policy), + ) .await .expect("create accepted"); @@ -1199,13 +1202,12 @@ mod lifecycle_tests { format!("$null = '{share}'; Start-Sleep -Seconds 60"), ]; let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); + let policy = fs_policy(&[&share]); backend - .policy_sink() - .lock() - .await - .insert("sb-stop".into(), fs_policy(&[&share])); - backend - .create_sandbox(&driver_sandbox_with_command("sb-stop", "", command)) + .create_sandbox( + &driver_sandbox_with_command("sb-stop", "", command), + Some(&policy), + ) .await .expect("create accepted"); wait_for(&backend, "sb-stop", |sandbox| { @@ -1245,13 +1247,8 @@ mod lifecycle_tests { binaries: Vec::new(), }, ); - backend - .policy_sink() - .lock() - .await - .insert("sb-net".into(), policy); let error = backend - .create_sandbox(&driver_sandbox("sb-net")) + .create_sandbox(&driver_sandbox("sb-net"), Some(&policy)) .await .expect_err("unmappable policy must fail CreateSandbox synchronously"); assert_eq!(error.code(), tonic::Code::InvalidArgument); diff --git a/crates/openshell-driver-mxc/src/grpc.rs b/crates/openshell-driver-mxc/src/grpc.rs index 9e7dada94f..17993a01a0 100644 --- a/crates/openshell-driver-mxc/src/grpc.rs +++ b/crates/openshell-driver-mxc/src/grpc.rs @@ -56,11 +56,12 @@ impl ComputeDriver for ComputeDriverService { &self, request: Request, ) -> Result, Status> { + let request = request.into_inner(); let sandbox = request - .into_inner() .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.backend.validate_sandbox_create(&sandbox)?; + self.backend + .validate_sandbox_create(&sandbox, request.policy.as_ref())?; Ok(Response::new(ValidateSandboxCreateResponse {})) } @@ -99,11 +100,13 @@ impl ComputeDriver for ComputeDriverService { &self, request: Request, ) -> Result, Status> { + let request = request.into_inner(); let sandbox = request - .into_inner() .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.backend.create_sandbox(&sandbox).await?; + self.backend + .create_sandbox(&sandbox, request.policy.as_ref()) + .await?; Ok(Response::new(CreateSandboxResponse {})) } diff --git a/crates/openshell-driver-mxc/src/policy.rs b/crates/openshell-driver-mxc/src/policy.rs index 1f26d186e5..a909238851 100644 --- a/crates/openshell-driver-mxc/src/policy.rs +++ b/crates/openshell-driver-mxc/src/policy.rs @@ -78,8 +78,8 @@ fn format_loss(items: &[LossItem]) -> String { /// Translates an `OpenShell` `SandboxPolicy` into an MXC `ContainerConfig` /// fragment, returning a loss report of anything unrepresentable. pub trait PolicyMapper: Send + Sync { - /// `policy` is `None` only when the gateway failed to stage one (the MXC - /// path treats that as a hard error — the demo's whole point is enforcement). + /// `policy` is `None` only when the create request omitted it. The MXC path + /// treats that as a hard error because it cannot launch without enforcement. fn map(&self, policy: Option<&SandboxPolicy>, ctx: &MapCtx) -> Result; } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 5bc6ea03b3..0bcae6ddfa 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -515,6 +515,8 @@ impl PodmanComputeDriver { default_image: self.config.default_image.clone(), gateway_manages_lifecycle: true, supports_sandbox_authentication: false, + sandbox_runtime_control: + openshell_core::proto::compute::v1::SandboxRuntimeControl::Supervisor.into(), }) } diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index 222c617c2f..cb06e49ed7 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -508,6 +508,7 @@ mod tests { client .validate_sandbox_create(request_with_traceparent(ValidateSandboxCreateRequest { sandbox: None, + policy: None, })) .await .expect_err("missing sandbox should fail"); diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 129965188d..cffdd508f6 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -525,6 +525,8 @@ impl VmDriver { default_image: self.config.default_image.clone(), gateway_manages_lifecycle: true, supports_sandbox_authentication: false, + sandbox_runtime_control: + openshell_core::proto::compute::v1::SandboxRuntimeControl::Supervisor.into(), } } @@ -5779,6 +5781,7 @@ mod tests { client .validate_sandbox_create(request_with_traceparent(ValidateSandboxCreateRequest { sandbox: None, + policy: None, })) .await .is_err() @@ -5787,6 +5790,7 @@ mod tests { client .create_sandbox(request_with_traceparent(CreateSandboxRequest { sandbox: None, + policy: None, })) .await .is_err() @@ -5894,6 +5898,7 @@ mod tests { }; let request = request_with_traceparent(CreateSandboxRequest { sandbox: Some(sandbox), + policy: None, }); let mut client = traced_driver_client(driver.clone()).await; diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index ee2beacd5f..c5a7ebe0e7 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -96,14 +96,10 @@ impl openshell_server::ComputeDriverFactory for MxcFactory { ) -> openshell_core::Result { let config: openshell_driver_mxc::MxcComputeConfig = context.driver_config()?; let backend = openshell_driver_mxc::MxcComputeBackend::new(config); - let sink = backend.policy_sink(); let driver = openshell_driver_mxc::ComputeDriverService::new(backend); - Ok( - openshell_server::ComputeDriverInstance::InProcessWithSandboxPolicy { - driver: std::sync::Arc::new(driver), - sink, - }, - ) + Ok(openshell_server::ComputeDriverInstance::InProcess( + std::sync::Arc::new(driver), + )) } } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index fa3e0f2c11..03da29055c 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -19,8 +19,6 @@ use crate::tracing_bus::TracingLogBus; use futures::{Stream, StreamExt}; #[cfg(unix)] use hyper_util::rt::TokioIo; -#[cfg(target_os = "windows")] -use openshell_core::proto::SandboxPolicy; use openshell_core::proto::compute::v1::{ AuthenticateSandboxRequest, CreateSandboxRequest, DeleteSandboxRequest, DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition, DriverPlatformEvent, DriverResourceRequirements, @@ -30,8 +28,8 @@ use openshell_core::proto::compute::v1::{ GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GpuResourceRequirements as DriverGpuResourceRequirements, ListSandboxesRequest, ResourceRequirements as DriverSandboxResourceRequirements, - StartSandboxRequest, StopSandboxRequest, ValidateSandboxCreateRequest, WatchSandboxesEvent, - WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, + SandboxRuntimeControl, StartSandboxRequest, StopSandboxRequest, ValidateSandboxCreateRequest, + WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, watch_sandboxes_event, }; @@ -265,6 +263,17 @@ pub struct ComputeDriverInfoSnapshot { pub gateway_manages_lifecycle: bool, /// Whether the driver authenticates driver-native sandbox credentials. pub supports_sandbox_authentication: bool, + /// Component responsible for runtime readiness and policy enforcement. + pub sandbox_runtime_control: SandboxRuntimeControl, +} + +fn normalize_sandbox_runtime_control(value: i32) -> SandboxRuntimeControl { + match SandboxRuntimeControl::try_from(value) { + Ok(SandboxRuntimeControl::Driver) => SandboxRuntimeControl::Driver, + Ok(SandboxRuntimeControl::Unspecified | SandboxRuntimeControl::Supervisor) | Err(_) => { + SandboxRuntimeControl::Supervisor + } + } } /// Interval between store-vs-backend reconciliation sweeps. @@ -564,10 +573,6 @@ pub struct ComputeRuntime { lifecycle_gates: Arc, gateway_listener_requirements: Vec, replica_id: String, - /// Optional policy side channel supplied by an in-process driver whose - /// public RPC contract cannot carry the typed sandbox policy. - #[cfg(target_os = "windows")] - sandbox_policy_sink: Option>>>, } impl fmt::Debug for ComputeRuntime { @@ -616,6 +621,9 @@ impl ComputeRuntime { driver_version: capabilities.driver_version, gateway_manages_lifecycle: capabilities.gateway_manages_lifecycle, supports_sandbox_authentication: capabilities.supports_sandbox_authentication, + sandbox_runtime_control: normalize_sandbox_runtime_control( + capabilities.sandbox_runtime_control, + ), }; let default_image = capabilities.default_image; let gateway_listener_requirements = match driver @@ -686,8 +694,6 @@ impl ComputeRuntime { lifecycle_gates: Arc::new(LifecycleGateRegistry::default()), gateway_listener_requirements, replica_id: lease::replica_id(), - #[cfg(target_os = "windows")] - sandbox_policy_sink: None, }) } @@ -758,6 +764,11 @@ impl ComputeRuntime { self.driver_info.supports_sandbox_authentication } + #[must_use] + pub(crate) fn sandbox_runtime_control(&self) -> SandboxRuntimeControl { + self.driver_info.sandbox_runtime_control + } + pub(crate) async fn authenticate_sandbox(&self, credential: &str) -> Result { if !self.supports_sandbox_authentication() { return Err(Status::unimplemented( @@ -789,16 +800,6 @@ impl ComputeRuntime { self } - #[cfg(target_os = "windows")] - #[must_use] - pub(crate) fn with_sandbox_policy_sink( - mut self, - sink: Arc>>, - ) -> Self { - self.sandbox_policy_sink = Some(sink); - self - } - #[must_use] pub(crate) fn gateway_listener_requirements(&self) -> &[GatewayListenerRequirement] { &self.gateway_listener_requirements @@ -841,6 +842,7 @@ impl ComputeRuntime { pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { let driver_sandbox = driver_sandbox_from_public(sandbox, &self.driver_info.name) .map_err(|status| *status)?; + let policy = sandbox.spec.as_ref().and_then(|spec| spec.policy.clone()); self.driver .call( "driver.validate_sandbox_create", @@ -849,6 +851,7 @@ impl ComputeRuntime { driver .validate_sandbox_create(Request::new(ValidateSandboxCreateRequest { sandbox: Some(driver_sandbox), + policy, })) .await }, @@ -866,6 +869,7 @@ impl ComputeRuntime { let sandbox_id = sandbox.object_id().to_string(); let mut driver_sandbox = driver_sandbox_from_public(&sandbox, &self.driver_info.name) .map_err(|status| *status)?; + let policy = sandbox.spec.as_ref().and_then(|spec| spec.policy.clone()); // Create with MustCreate condition to prevent duplicate creation race self.sandbox_index.update_from_sandbox(&sandbox); @@ -913,16 +917,6 @@ impl ComputeRuntime { if let Some(spec) = driver_sandbox.spec.as_mut() { spec.await_main_process_attachment = await_main_process_attachment; } - // A1: stage the typed SandboxPolicy out-of-band into the MXC backend's - // side channel, keyed by sandbox id (== DriverSandbox.id), immediately - // before dispatch. The driver removes/consumes it in create_sandbox. The - // proto driver contract has no policy field, so this is the only path. - #[cfg(target_os = "windows")] - if let Some(sink) = &self.sandbox_policy_sink - && let Some(p) = sandbox.spec.as_ref().and_then(|s| s.policy.clone()) - { - sink.lock().await.insert(sandbox_id.clone(), p); - } match self .driver .call( @@ -932,6 +926,7 @@ impl ComputeRuntime { driver .create_sandbox(Request::new(CreateSandboxRequest { sandbox: Some(driver_sandbox), + policy, })) .await }, @@ -1348,7 +1343,12 @@ impl ComputeRuntime { match self .store .update_message_cas::(&sandbox_id, expected_resource_version, |sandbox| { - apply_driver_snapshot(sandbox, snapshot, session_connected); + apply_driver_snapshot( + sandbox, + snapshot, + session_connected, + self.driver_info.sandbox_runtime_control, + ); }) .await { @@ -1796,7 +1796,14 @@ impl ComputeRuntime { sandbox_id, deleting_resource_version, "reconcile observed backend snapshot", - |sandbox| apply_driver_snapshot(sandbox, &snapshot, session_connected), + |sandbox| { + apply_driver_snapshot( + sandbox, + &snapshot, + session_connected, + self.driver_info.sandbox_runtime_control, + ); + }, ) .await; } @@ -2836,7 +2843,14 @@ impl ComputeRuntime { .update_message_cas::( &incoming.id, expected_resource_version, - |sandbox| apply_driver_snapshot(sandbox, &incoming, session_connected), + |sandbox| { + apply_driver_snapshot( + sandbox, + &incoming, + session_connected, + self.driver_info.sandbox_runtime_control, + ); + }, ) .await .map_err(|e| match e { @@ -4020,7 +4034,12 @@ fn public_status_from_driver( } } -fn apply_driver_snapshot(sandbox: &mut Sandbox, incoming: &DriverSandbox, session_connected: bool) { +fn apply_driver_snapshot( + sandbox: &mut Sandbox, + incoming: &DriverSandbox, + session_connected: bool, + sandbox_runtime_control: SandboxRuntimeControl, +) { let old_phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); let sandbox_name = &incoming.name; @@ -4055,7 +4074,8 @@ fn apply_driver_snapshot(sandbox: &mut Sandbox, incoming: &DriverSandbox, sessio (phase, status) }, |incoming_status| { - let composed = ComposedPhase::new(incoming_status, session_connected); + let composed = + ComposedPhase::new(incoming_status, session_connected, sandbox_runtime_control); let mut status = Some(public_status_from_driver( incoming_status, composed.phase, @@ -4206,7 +4226,11 @@ struct ComposedPhase { } impl ComposedPhase { - fn new(incoming_status: &DriverSandboxStatus, session_connected: bool) -> Self { + fn new( + incoming_status: &DriverSandboxStatus, + session_connected: bool, + sandbox_runtime_control: SandboxRuntimeControl, + ) -> Self { let backend_phase = derive_phase(Some(incoming_status)); // A live supervisor session is a stronger readiness signal than the backend phase. // set_supervisor_session_state may have already promoted the store record to Ready @@ -4214,13 +4238,17 @@ impl ComposedPhase { // backend phase overwrite it. let phase = match backend_phase { SandboxPhase::Error | SandboxPhase::Deleting | SandboxPhase::Stopped => backend_phase, + SandboxPhase::Ready if sandbox_runtime_control == SandboxRuntimeControl::Driver => { + SandboxPhase::Ready + } _ if session_connected => SandboxPhase::Ready, _ => SandboxPhase::Provisioning, }; Self { phase, session_connected, - backend_ready_without_session: backend_phase == SandboxPhase::Ready + backend_ready_without_session: sandbox_runtime_control != SandboxRuntimeControl::Driver + && backend_phase == SandboxPhase::Ready && !session_connected, } } @@ -4504,6 +4532,7 @@ impl ComputeDriver for NoopTestDriver { default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: self.sandbox_authentication.is_some(), + sandbox_runtime_control: SandboxRuntimeControl::Supervisor.into(), }, )) } @@ -4647,6 +4676,7 @@ pub async fn new_test_runtime_with_driver( driver_version: "test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication, + sandbox_runtime_control: SandboxRuntimeControl::Supervisor, }, telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process: None, @@ -4660,8 +4690,6 @@ pub async fn new_test_runtime_with_driver( lifecycle_gates: Arc::new(LifecycleGateRegistry::default()), gateway_listener_requirements: Vec::new(), replica_id: "test-replica".to_string(), - #[cfg(target_os = "windows")] - sandbox_policy_sink: None, } } @@ -4826,6 +4854,7 @@ mod tests { default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: false, + sandbox_runtime_control: SandboxRuntimeControl::Supervisor.into(), })) } @@ -5166,6 +5195,7 @@ mod tests { default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: false, + sandbox_runtime_control: SandboxRuntimeControl::Supervisor.into(), })) } @@ -5375,6 +5405,7 @@ mod tests { driver_version: "test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: false, + sandbox_runtime_control: SandboxRuntimeControl::Supervisor, }, telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process: None, @@ -5388,8 +5419,6 @@ mod tests { lifecycle_gates: Arc::new(LifecycleGateRegistry::default()), gateway_listener_requirements: Vec::new(), replica_id: "test-replica".to_string(), - #[cfg(target_os = "windows")] - sandbox_policy_sink: None, } } @@ -6032,6 +6061,34 @@ mod tests { assert!(store.is_single_replica()); } + #[test] + fn unspecified_runtime_control_preserves_supervisor_compatibility() { + assert_eq!( + normalize_sandbox_runtime_control(SandboxRuntimeControl::Unspecified.into()), + SandboxRuntimeControl::Supervisor + ); + assert_eq!( + normalize_sandbox_runtime_control(i32::MAX), + SandboxRuntimeControl::Supervisor + ); + } + + #[test] + fn driver_controlled_runtime_uses_driver_readiness() { + let status = make_driver_status(DriverCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: "AgentRunning".to_string(), + message: "MXC workload is running".to_string(), + last_transition_time: String::new(), + }); + + let composed = ComposedPhase::new(&status, false, SandboxRuntimeControl::Driver); + + assert_eq!(composed.phase, SandboxPhase::Ready); + assert!(!composed.backend_ready_without_session); + } + #[test] fn delete_gate_registry_removes_stale_entries() { let registry = LifecycleGateRegistry::default(); @@ -10337,12 +10394,14 @@ mod tests { remote .validate_sandbox_create(Request::new(ValidateSandboxCreateRequest { sandbox: Some(sandbox.clone()), + policy: None, })) .await .unwrap(); remote .create_sandbox(Request::new(CreateSandboxRequest { sandbox: Some(sandbox.clone()), + policy: None, })) .await .unwrap(); @@ -10482,6 +10541,10 @@ mod tests { let mut sandbox = sandbox_record("sb-uds", "uds-sandbox", SandboxPhase::Provisioning); sandbox.spec = Some(SandboxSpec { log_level: "debug".to_string(), + policy: Some(openshell_core::proto::SandboxPolicy { + version: 42, + ..Default::default() + }), template: Some(SandboxTemplate { image: "ghcr.io/nvidia/openshell/sandbox:test".to_string(), driver_config: Some(prost_types::Struct { @@ -10510,7 +10573,11 @@ mod tests { let validated = match &calls[2] { FakeComputeDriverCall::ValidateSandboxCreate { sandbox: Some(sandbox), - } => sandbox, + policy: Some(policy), + } => { + assert_eq!(policy.version, 42); + sandbox + } other => panic!("expected ValidateSandboxCreate call, got {other:?}"), }; let driver_config = validated @@ -10521,6 +10588,13 @@ mod tests { .expect("selected driver_config should be forwarded"); assert!(driver_config.fields.contains_key("pool")); assert!(!driver_config.fields.contains_key("network_mode")); + assert!(matches!( + &calls[3], + FakeComputeDriverCall::CreateSandbox { + policy: Some(policy), + .. + } if policy.version == 42 + )); driver.clear_calls(); runtime diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index abb59fadde..fc140fec71 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -23,6 +23,7 @@ use crate::provider_profile_sources::EffectiveProviderProfileCatalog; #[cfg(test)] use crate::provider_profile_sources::ProviderProfileSources; use openshell_core::net::{is_always_blocked_ip, is_internal_ip}; +use openshell_core::proto::compute::v1::SandboxRuntimeControl; use openshell_core::proto::policy_merge_operation; use openshell_core::proto::setting_value; use openshell_core::proto::{ @@ -3136,13 +3137,13 @@ pub(super) async fn handle_get_sandbox_provider_environment( // --------------------------------------------------------------------------- fn validate_live_policy_update_support( - driver_name: &str, + sandbox_runtime_control: SandboxRuntimeControl, has_policy: bool, has_merge_ops: bool, ) -> Result<(), Status> { - if (has_policy || has_merge_ops) && driver_name == "mxc" { + if (has_policy || has_merge_ops) && sandbox_runtime_control == SandboxRuntimeControl::Driver { return Err(Status::failed_precondition( - "live policy updates are not supported for MXC sandboxes; recreate the sandbox so the new policy is mapped before launch", + "live policy updates are not supported when the selected compute driver owns runtime policy enforcement; recreate the sandbox to apply the new policy", )); } Ok(()) @@ -3223,7 +3224,7 @@ async fn handle_update_config_inner( )); } validate_live_policy_update_support( - state.compute.configured_driver_name(), + state.compute.sandbox_runtime_control(), has_policy, has_merge_ops, )?; @@ -6915,17 +6916,23 @@ mod tests { use tonic::Code; #[test] - fn mxc_rejects_sandbox_policy_replacement_and_merge_updates() { + fn driver_controlled_runtime_rejects_sandbox_policy_replacement_and_merge_updates() { for (has_policy, has_merge_ops) in [(true, false), (false, true)] { - let error = validate_live_policy_update_support("mxc", has_policy, has_merge_ops) - .expect_err("MXC must reject policy mutations after launch"); + let error = validate_live_policy_update_support( + SandboxRuntimeControl::Driver, + has_policy, + has_merge_ops, + ) + .expect_err("driver-controlled runtimes must reject policy mutations after launch"); assert_eq!(error.code(), Code::FailedPrecondition); } - let error = validate_live_policy_update_support("mxc", true, false).expect_err( - "global policy replacement also changes desired state for live MXC sandboxes", - ); + let error = validate_live_policy_update_support(SandboxRuntimeControl::Driver, true, false) + .expect_err("global policy replacement also changes desired state for live sandboxes"); assert_eq!(error.code(), Code::FailedPrecondition); + + validate_live_policy_update_support(SandboxRuntimeControl::Supervisor, true, false) + .expect("supervisor-controlled runtimes support live policy replacement"); } /// Wrap a request with a user `Principal` so handler scope guards treat diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 0b888b3609..2acd34fc44 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1051,12 +1051,6 @@ pub use compute::{ pub enum ComputeDriverInstance { /// A driver hosted in the gateway process. InProcess(SharedComputeDriver), - /// An in-process driver with a typed sandbox-policy side channel. - #[cfg(target_os = "windows")] - InProcessWithSandboxPolicy { - driver: SharedComputeDriver, - sink: Arc>>, - }, /// A driver process launched and owned by the gateway. ManagedRemote(AcquiredRemoteDriverEndpoint), } @@ -1455,24 +1449,6 @@ async fn build_compute_runtime( .map_err(|error| { Error::execution(format!("failed to create compute runtime: {error}")) })?, - #[cfg(target_os = "windows")] - ComputeDriverInstance::InProcessWithSandboxPolicy { driver, sink } => { - ComputeRuntime::from_driver( - registration.name, - driver, - None, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - .map_err(|error| { - Error::execution(format!("failed to create compute runtime: {error}")) - })? - .with_sandbox_policy_sink(sink) - } ComputeDriverInstance::ManagedRemote(mut endpoint) => { endpoint.name = registration.name; ComputeRuntime::new_remote_driver( diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index 5f4063ae45..6b786699c4 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -4,6 +4,7 @@ //! Test fixtures for exercising gateway integration points. use futures::{Stream, stream}; +use openshell_core::proto::SandboxPolicy; #[cfg(unix)] use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_core::proto::compute::v1::{ @@ -40,6 +41,7 @@ pub enum FakeComputeDriverCall { GetGatewayListenerRequirements, ValidateSandboxCreate { sandbox: Option, + policy: Option, }, GetSandbox { sandbox_id: String, @@ -48,6 +50,7 @@ pub enum FakeComputeDriverCall { ListSandboxes, CreateSandbox { sandbox: Option, + policy: Option, }, StopSandbox { sandbox_id: String, @@ -96,6 +99,8 @@ impl FakeComputeDriver { default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: false, + sandbox_runtime_control: + openshell_core::proto::compute::v1::SandboxRuntimeControl::Supervisor.into(), }, gateway_listener_requirements: Vec::new(), gateway_listener_requirements_supported: true, @@ -284,11 +289,14 @@ impl ComputeDriver for FakeComputeDriver { request: Request, ) -> Result, Status> { self.record_traceparent(request.metadata()); - let sandbox = request.into_inner().sandbox; + let request = request.into_inner(); self.with_state(|state| { state .calls - .push(FakeComputeDriverCall::ValidateSandboxCreate { sandbox }); + .push(FakeComputeDriverCall::ValidateSandboxCreate { + sandbox: request.sandbox, + policy: request.policy, + }); }); Ok(Response::new(ValidateSandboxCreateResponse {})) } @@ -337,14 +345,16 @@ impl ComputeDriver for FakeComputeDriver { request: Request, ) -> Result, Status> { self.record_traceparent(request.metadata()); - let sandbox = request.into_inner().sandbox; + let request = request.into_inner(); + let sandbox = request.sandbox; self.with_state(|state| { if let Some(sandbox) = sandbox.as_ref() { state.sandboxes.insert(sandbox.id.clone(), sandbox.clone()); } - state - .calls - .push(FakeComputeDriverCall::CreateSandbox { sandbox }); + state.calls.push(FakeComputeDriverCall::CreateSandbox { + sandbox, + policy: request.policy, + }); }); Ok(Response::new(CreateSandboxResponse {})) } diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index ffa18f1b34..0680d208f3 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -3,14 +3,20 @@ # SPDX-License-Identifier: Apache-2.0 title: "Sandbox Compute Drivers" sidebar-title: "Compute Drivers" -description: "Reference for Docker, Podman, MicroVM, and Kubernetes sandbox compute drivers." -keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Docker, Podman, MicroVM, Kubernetes, Reference" +description: "Reference for Docker, Podman, MicroVM, Kubernetes, and Windows MXC sandbox compute drivers." +keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Docker, Podman, MicroVM, Kubernetes, MXC, Reference" position: 4 --- The gateway's configured compute driver determines how OpenShell creates each sandbox. The CLI workflow stays the same across drivers: you create, connect to, inspect, stop, start, and delete sandboxes through the gateway API. -Every compute driver runs the OpenShell supervisor inside the sandbox workload. The supervisor launches the agent process, applies policy, routes egress through the proxy, injects configured credentials, and maintains the gateway session. +Most compute drivers run the OpenShell supervisor inside the sandbox workload. +The supervisor launches the agent process, applies policy, routes egress through +the proxy, injects configured credentials, and maintains the gateway session. +A driver may instead advertise driver-owned runtime control. In that mode, the +gateway sends the canonical policy in the validation and creation requests, the +driver applies it before launch, and driver-reported readiness does not require +a supervisor session. The Windows MXC driver uses this mode. Stop stops compute but retains the sandbox record and the driver's persistent workspace boundary. Start reactivates the same driver resource. @@ -42,7 +48,8 @@ Configure the compute driver on the gateway. Current releases accept one driver compute_drivers = ["docker"] ``` -Reserved built-in values are `docker`, `podman`, `kubernetes`, and `vm`. +Reserved built-in values are `docker`, `podman`, `kubernetes`, `vm`, and `mxc`. +The `mxc` driver is available only in native Windows gateway builds. Non-reserved names select an extension driver and require a `socket_path` in `[openshell.drivers.]`. diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index 16a63c0db5..e33834b824 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -7,6 +7,7 @@ package openshell.compute.v1; import "google/protobuf/struct.proto"; import "options.proto"; +import "sandbox.proto"; // Gateway/compute-driver extension contract. // @@ -71,6 +72,17 @@ service ComputeDriver { message GetCapabilitiesRequest {} +// Component responsible for the sandbox runtime control plane. +enum SandboxRuntimeControl { + // Compatibility default for drivers that predate this capability. The + // gateway treats this as SANDBOX_RUNTIME_CONTROL_SUPERVISOR. + SANDBOX_RUNTIME_CONTROL_UNSPECIFIED = 0; + // The standard OpenShell supervisor owns readiness and live policy updates. + SANDBOX_RUNTIME_CONTROL_SUPERVISOR = 1; + // The compute driver owns readiness and applies policy while provisioning. + SANDBOX_RUNTIME_CONTROL_DRIVER = 2; +} + message GetCapabilitiesResponse { reserved 4, 5; reserved "supports_gpu", "gpu_count"; @@ -86,6 +98,8 @@ message GetCapabilitiesResponse { bool gateway_manages_lifecycle = 6; // Whether AuthenticateSandbox is implemented by this driver. bool supports_sandbox_authentication = 7; + // Component responsible for runtime readiness and policy enforcement. + SandboxRuntimeControl sandbox_runtime_control = 8; } message AuthenticateSandboxRequest { @@ -292,6 +306,9 @@ message DriverPlatformEvent { message ValidateSandboxCreateRequest { // Proposed sandbox configuration to validate before provisioning. DriverSandbox sandbox = 1; + // Canonical policy the selected driver must validate when it owns runtime + // control. Supervisor-controlled drivers may ignore this field. + openshell.sandbox.v1.SandboxPolicy policy = 2; } message ValidateSandboxCreateResponse {} @@ -318,6 +335,9 @@ message ListSandboxesResponse { message CreateSandboxRequest { // Sandbox configuration to provision on the compute platform. DriverSandbox sandbox = 1; + // Canonical policy the selected driver must apply when it owns runtime + // control. Supervisor-controlled drivers may ignore this field. + openshell.sandbox.v1.SandboxPolicy policy = 2; } message CreateSandboxResponse {} From ee76daf7a412e5ffb839b570d7206a54af287982 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 31 Aug 2026 23:17:56 -0700 Subject: [PATCH 9/9] refactor(compute): separate policy delivery from readiness Signed-off-by: Drew Newberry --- .agents/skills/openshell-cli/SKILL.md | 4 +- .agents/skills/openshell-cli/cli-reference.md | 4 +- architecture/README.md | 13 ++- architecture/compute-runtimes.md | 33 +++--- architecture/windows-msvc-build.md | 4 +- crates/openshell-driver-docker/src/lib.rs | 3 +- crates/openshell-driver-docker/src/tests.rs | 11 +- .../openshell-driver-kubernetes/src/driver.rs | 3 +- .../openshell-driver-kubernetes/src/grpc.rs | 5 +- crates/openshell-driver-mxc/README.md | 12 +- crates/openshell-driver-mxc/src/driver.rs | 55 ++++----- crates/openshell-driver-mxc/src/grpc.rs | 7 +- crates/openshell-driver-podman/src/driver.rs | 3 +- crates/openshell-driver-podman/src/grpc.rs | 1 - crates/openshell-driver-vm/src/driver.rs | 6 +- crates/openshell-server/src/compute/mod.rs | 109 +++++++----------- crates/openshell-server/src/grpc/policy.rs | 39 ------- crates/openshell-server/src/test_support.rs | 14 +-- docs/reference/sandbox-compute-drivers.mdx | 9 +- proto/compute_driver.proto | 27 ++--- 20 files changed, 126 insertions(+), 236 deletions(-) diff --git a/.agents/skills/openshell-cli/SKILL.md b/.agents/skills/openshell-cli/SKILL.md index a1ed3bef9b..c0be7fc730 100644 --- a/.agents/skills/openshell-cli/SKILL.md +++ b/.agents/skills/openshell-cli/SKILL.md @@ -384,7 +384,7 @@ the operation that removes retained state. This is the most important multi-step workflow. It enables a tight feedback cycle where sandbox policy is refined based on observed activity. -**Key concept**: Policies have static fields (immutable after creation: `filesystem_policy`, `landlock`, `process`) and two dynamic fields: `network_policies` and `network_middlewares`. Both dynamic fields can be updated without recreating the sandbox when the selected compute driver supports live policy updates. MXC rejects live policy replacement and merge updates; delete and recreate an MXC sandbox instead. +**Key concept**: Policies have static fields (immutable after creation: `filesystem_policy`, `landlock`, `process`) and two dynamic fields: `network_policies` and `network_middlewares`. Both dynamic fields can be updated without recreating the sandbox when the selected compute driver supports live policy updates. Drivers without the standard supervisor fetch revisions through the sandbox configuration API and report whether they loaded them. An endpoint with omitted `protocol` retains explicit-proxy behavior. Explicit `protocol: tcp` requests policy DNS and transparent TCP and currently requires @@ -454,7 +454,7 @@ Edit `current-policy.yaml` to allow the blocked actions. **For policy content au - Binary matching patterns - Ordered `network_middlewares`, host selection, HTTP and WebSocket bindings, and `fail_open` or `fail_closed` behavior -`network_policies` and `network_middlewares` can be modified at runtime when the selected compute driver supports live policy updates. MXC rejects live policy replacement and merge updates; delete and recreate an MXC sandbox instead. If `filesystem_policy`, `landlock`, or `process` need changes, the sandbox must be recreated. Built-in middleware such as `openshell/regex` needs no gateway registration. An operator-run middleware must already be registered under `[[openshell.supervisor.middleware]]`; changing that static registration requires a gateway restart. +`network_policies` and `network_middlewares` can be modified at runtime when the selected compute driver supports live policy updates. Use `--wait` to verify that the active runtime loaded the revision; do not infer enforcement from the gateway accepting the update. If `filesystem_policy`, `landlock`, or `process` need changes, the sandbox must be recreated. Built-in middleware such as `openshell/regex` needs no gateway registration. An operator-run middleware must already be registered under `[[openshell.supervisor.middleware]]`; changing that static registration requires a gateway restart. Middleware can inspect parsed HTTP request bodies and complete client-to-upstream WebSocket text messages over both `ws://` and `wss://` when the implementation advertises the matching binding. The built-in `openshell/regex` advertises both bindings and applies its fixed patterns to UTF-8 text. A host-matched HTTP-only attachment can inspect the upgrade GET but does not join the WebSocket chain; look for `binding_not_selected` coverage. Binary messages pass under both `on_error` modes and active stages emit `unsupported_message_type` coverage; upstream-to-client messages remain uninspected. A broken fail-open WebSocket stage is disabled for the rest of that connection; inspect sandbox OCSF logs for `openshell.middleware.websocket_stage_disabled`. diff --git a/.agents/skills/openshell-cli/cli-reference.md b/.agents/skills/openshell-cli/cli-reference.md index f53289b89f..f84fa85fa4 100644 --- a/.agents/skills/openshell-cli/cli-reference.md +++ b/.agents/skills/openshell-cli/cli-reference.md @@ -413,7 +413,7 @@ The sandbox name defaults to the last-used sandbox. ### `openshell policy update [name]` -Incrementally merge live network policy changes into the current sandbox policy when the selected compute driver supports live updates. Multiple flags in one invocation are applied as one atomic batch and create at most one new revision. MXC rejects live policy merges; delete and recreate an MXC sandbox instead. +Incrementally merge live network policy changes into the current sandbox policy when the selected compute driver supports live updates. Multiple flags in one invocation are applied as one atomic batch and create at most one new revision. Use `--wait` to verify that the active runtime loaded the new revision. | Flag | Default | Description | |------|---------|-------------| @@ -443,7 +443,7 @@ Notes: ### `openshell policy set [name] --policy ` -Replace the full policy on a live sandbox when the selected compute driver supports live updates. Only the dynamic `network_policies` field can be changed at runtime. MXC rejects live policy replacement; delete and recreate an MXC sandbox instead. +Replace the full policy on a live sandbox when the selected compute driver supports live updates. Only dynamic policy fields can be changed at runtime. Use `--wait` to verify that the active runtime loaded the new revision. | Flag | Default | Description | |------|---------|-------------| diff --git a/architecture/README.md b/architecture/README.md index 5b77af1f8d..d48a0d53f9 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -141,12 +141,13 @@ bridge networks, port mappings, NAT traversal, or bespoke tunnels. The common runtime requirement is narrower: the supervisor must be able to reach the gateway. -The compute-driver capability contract identifies whether the supervisor or the -driver owns runtime control. Supervisor-controlled drivers use the session model -above. A driver-controlled runtime receives the canonical policy in its validate -and create requests, applies it before launch, and self-reports readiness without -a supervisor session. The Windows MXC driver uses this mode and does not expose -interactive connect, live policy delivery, or governed egress. +The compute-driver capability contract identifies whether a driver reports +runtime readiness. Most drivers use the supervisor session model above. A +driver that sets `driver_reports_runtime_readiness` may self-report readiness +without a supervisor session. Every driver receives the canonical create-time +policy in `DriverSandboxSpec`; drivers without a supervisor use the existing +sandbox configuration API for later revisions. The Windows MXC driver reports +its own readiness and does not expose interactive connect or governed egress. The gateway delivers desired state; the sandbox applies it locally. Policy, settings, credentials, and inference routes flow from the gateway to the diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 89584ebc43..aa97fbcb78 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -15,8 +15,8 @@ is responsible for: - For supervisor-controlled runtimes, injecting sandbox identity and gateway callback configuration, supplying callback credentials, and providing the supervisor binary or image. -- For driver-controlled runtimes, validating and applying the canonical policy - before launching the workload. +- For runtimes without the standard supervisor, validating and applying the + canonical policy before launching the workload. - Forwarding the exact canonical main-process argv and TTY mode without shell reconstruction. The sandbox-level environment and policy workspace apply to the main process. @@ -27,16 +27,17 @@ Drivers report **runtime-observed state only** and must not hold references to gateway-internal types. For supervisor-controlled runtimes, `Ready=True` means only that the compute resource is healthy; the gateway also requires a supervisor session before publishing `SandboxPhase::Ready`. For -driver-controlled runtimes, `Ready=True` is authoritative because the driver +drivers that report runtime readiness, `Ready=True` is authoritative because the driver launches and monitors the policy-constrained workload itself. `compute_driver.proto` is the supported gateway/driver extension boundary. At initialization the gateway snapshots the driver's identity, version, -default image, gateway-lifecycle preference, and `sandbox_runtime_control` from -`GetCapabilities`. An omitted or unknown runtime-control value is treated as -`SUPERVISOR` for compatibility. The gateway includes the canonical -`SandboxPolicy` in both `ValidateSandboxCreate` and `CreateSandbox`; drivers -that advertise `DRIVER` must validate and apply it before launch. +default image, gateway-lifecycle preference, and +`driver_reports_runtime_readiness` from `GetCapabilities`. The gateway includes +the canonical `SandboxPolicy` in `DriverSandboxSpec.policy` for validation and +creation. Drivers that enforce policy outside the standard supervisor fetch +later revisions through `GetSandboxConfig` and acknowledge them through +`ReportPolicyStatus`. Process-identity omissions are preserved across this boundary so every driver can apply its native image or runtime defaults. Driver-requested listeners are structurally validated and remain restricted to sandbox callback RPCs. @@ -53,7 +54,7 @@ reason strings. ## Sandbox Readiness Composition -The gateway composes driver state with the advertised runtime-control mode to +The gateway composes driver state with the advertised readiness behavior to produce the public `SandboxPhase`: ``` @@ -61,7 +62,7 @@ backend_phase = derive_phase(driver_status) public_phase = if backend_phase in {Error, Deleting}: → pass through (terminal precedence) - if runtime_control == Driver && backend_phase == Ready: → Ready + if driver_reports_runtime_readiness && backend_phase == Ready: → Ready if backend_phase == Ready && session connected: → Ready if backend_phase == Ready && no session: → Provisioning if backend_phase in {Provisioning, Unknown} && session: → Ready @@ -74,7 +75,7 @@ backend reports ready but has no supervisor session yet holds `Provisioning` wit `Ready=False`, `SupervisorNotConnected` condition and the message `Backend ready; waiting for supervisor session`. This distinguishes it from a sandbox whose compute resource is still provisioning without exposing contradictory public -readiness signals. For a driver-controlled runtime, the driver's ready condition +readiness signals. When the driver reports runtime readiness, its ready condition is published without waiting for a supervisor session. **Session precedence over lagging driver snapshots:** A supervisor session can only be @@ -94,11 +95,11 @@ session-owning replica. That work is deferred to GitHub issue #1868. Until then, deployments that require reliable readiness composition must run a single gateway replica. -**Extension point:** Runtime control is a driver capability, not an -operator-configurable hook. A driver may advertise `DRIVER` only when it applies -the canonical create-time policy and owns workload readiness. Driver-controlled -runtimes do not support live policy mutation; recreate the sandbox to apply a -new policy. RFC-0010 lifecycle hooks may observe readiness transitions via +**Extension point:** Driver-reported readiness is a capability, not an +operator-configurable hook. A driver may enable it only when it owns workload +readiness. Policy delivery remains independent: create-time policy is embedded +in the sandbox specification, and later revisions use the existing sandbox +configuration API. RFC-0010 lifecycle hooks may observe readiness transitions via `post_commit`; they do not override the composition rule. The capability RPC reports driver identity, version, and the default sandbox diff --git a/architecture/windows-msvc-build.md b/architecture/windows-msvc-build.md index 8ee6c0e444..6b0d675a9c 100644 --- a/architecture/windows-msvc-build.md +++ b/architecture/windows-msvc-build.md @@ -11,8 +11,8 @@ driver. It does not make Windows a Docker, Kubernetes, Podman, or VM runtime hos - Preserve gateway configuration parsing for all existing compute driver names. - Build and test the in-process MXC driver on supported Windows hosts. - Use the ordinary in-process compute-driver composition path; MXC receives the - canonical sandbox policy through validate/create RPC messages and advertises - driver-owned runtime control for readiness. + canonical sandbox policy through `DriverSandboxSpec` and advertises that it + reports runtime readiness. - Return clear unsupported errors when a Windows gateway is configured to use Docker, Kubernetes, Podman, or VM. - Keep dedicated `windows:*` validation tasks while allowing the repository-wide `pre-commit` task to delegate compiler-bearing Rust checks to the native diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 147ccc7aec..d599cac169 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -667,8 +667,7 @@ impl DockerComputeDriver { default_image: self.config.default_image.clone(), gateway_manages_lifecycle: true, supports_sandbox_authentication: false, - sandbox_runtime_control: - openshell_core::proto::compute::v1::SandboxRuntimeControl::Supervisor.into(), + driver_reports_runtime_readiness: false, } } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index c8ab3f8082..07fb2b0ffb 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -43,6 +43,7 @@ fn test_sandbox() -> DriverSandbox { environment: HashMap::from([("TEMPLATE_ENV".to_string(), "template".to_string())]), ..Default::default() }), + policy: None, resource_requirements: None, sandbox_token: String::new(), command: Vec::new(), @@ -222,10 +223,7 @@ async fn tracing_in_process_service_preserves_the_driver_rpc_server_boundary() { ); ComputeDriver::validate_sandbox_create( &service, - Request::new(ValidateSandboxCreateRequest { - sandbox: None, - policy: None, - }), + Request::new(ValidateSandboxCreateRequest { sandbox: None }), ) .instrument(gateway_span) .await @@ -309,10 +307,7 @@ async fn tracing_lifecycle_rpc_failures_export_docker_operation_spans() { async { ComputeDriver::create_sandbox( &driver, - Request::new(CreateSandboxRequest { - sandbox: None, - policy: None, - }), + Request::new(CreateSandboxRequest { sandbox: None }), ) .await .expect_err("missing sandbox should fail"); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index c858f103ed..afe3f579e0 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -575,8 +575,7 @@ impl KubernetesComputeDriver { default_image: self.config.default_image.clone(), gateway_manages_lifecycle: false, supports_sandbox_authentication: true, - sandbox_runtime_control: - openshell_core::proto::compute::v1::SandboxRuntimeControl::Supervisor.into(), + driver_reports_runtime_readiness: false, }) } diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index dbf609cd12..027c350a26 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -438,10 +438,7 @@ mod tests { ComputeDriver::validate_sandbox_create( &service, - Request::new(ValidateSandboxCreateRequest { - sandbox: None, - policy: None, - }), + Request::new(ValidateSandboxCreateRequest { sandbox: None }), ) .await } diff --git a/crates/openshell-driver-mxc/README.md b/crates/openshell-driver-mxc/README.md index 7136e89a26..bd0b51f48b 100644 --- a/crates/openshell-driver-mxc/README.md +++ b/crates/openshell-driver-mxc/README.md @@ -5,10 +5,10 @@ OpenShell compute driver backed by **Microsoft MXC** (`wxc-exec`) on Windows. ## Design This driver implements the gateway's ordinary in-process `ComputeDriver` -contract and is linked into `openshell-gateway`. It advertises driver-owned -runtime control, so the gateway passes the canonical `SandboxPolicy` in both -`ValidateSandboxCreate` and `CreateSandbox` and accepts driver-reported -readiness without a supervisor session. `process_container` launches a one-shot +contract and is linked into `openshell-gateway`. It sets +`driver_reports_runtime_readiness`, so the gateway accepts driver-reported +readiness without a supervisor session. The canonical create-time +`SandboxPolicy` is carried by `DriverSandboxSpec.policy`. `process_container` launches a one-shot AppContainer and is the default. The opt-in `isolation_session` backend uses the state-aware `provision` → `start` → `exec` → `stop` → `deprovision` lifecycle. The driver launches and monitors the configured workload itself and self-reports @@ -54,7 +54,9 @@ openshell sandbox create --name mxc-demo --policy demo.yaml ` The `command` array is required and preserves Windows argument boundaries. `cwd` is optional. Environment variables come from the standard sandbox and template environment maps; the driver never copies values from the gateway host environment. -Network policy and live policy replacement or merge updates are rejected while the gateway uses MXC. Delete and recreate the sandbox to apply a different filesystem policy. +Network policy support depends on the selected containment path. Policy +replacement and merge updates use the gateway's standard sandbox configuration +revision contract. ## Prerequisites (live runs) diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index edd7bff3a0..1ded7130c4 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -11,9 +11,8 @@ use openshell_core::gpu::{driver_gpu_requirements, effective_driver_gpu_count}; use openshell_core::proto::SandboxPolicy; use openshell_core::proto::compute::v1::{ DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, - GetCapabilitiesResponse, SandboxRuntimeControl, WatchSandboxesDeletedEvent, - WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesSandboxEvent, - watch_sandboxes_event, + GetCapabilitiesResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, + WatchSandboxesPlatformEvent, WatchSandboxesSandboxEvent, watch_sandboxes_event, }; use openshell_core::proto_struct::struct_to_json_value; use serde::{Deserialize, Serialize}; @@ -291,7 +290,7 @@ impl MxcComputeBackend { default_image: DEFAULT_IMAGE_SENTINEL.to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: false, - sandbox_runtime_control: SandboxRuntimeControl::Driver.into(), + driver_reports_runtime_readiness: true, } } @@ -335,12 +334,9 @@ impl MxcComputeBackend { .map_err(|error| tonic::Status::invalid_argument(error.to_string())) } - pub fn validate_sandbox_create( - &self, - sandbox: &DriverSandbox, - policy: Option<&SandboxPolicy>, - ) -> Result<(), tonic::Status> { + pub fn validate_sandbox_create(&self, sandbox: &DriverSandbox) -> Result<(), tonic::Status> { self.validate_sandbox_fields(sandbox)?; + let policy = sandbox.spec.as_ref().and_then(|spec| spec.policy.as_ref()); self.map_sandbox_policy(&sandbox.id, policy)?; Ok(()) } @@ -357,11 +353,7 @@ impl MxcComputeBackend { registry.values().map(|e| e.sandbox.clone()).collect() } - pub async fn create_sandbox( - &self, - sandbox: &DriverSandbox, - policy: Option<&SandboxPolicy>, - ) -> Result<(), tonic::Status> { + pub async fn create_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), tonic::Status> { let sandbox_id = sandbox.id.clone(); self.validate_sandbox_fields(sandbox)?; @@ -370,6 +362,7 @@ impl MxcComputeBackend { // Policy translation is deterministic and side-effect free. Do it before // inserting the registry entry or launching MXC so invalid requests fail // synchronously at the CreateSandbox boundary. + let policy = sandbox.spec.as_ref().and_then(|spec| spec.policy.as_ref()); let mapped = self.map_sandbox_policy(&sandbox_id, policy)?; if sandbox @@ -947,6 +940,11 @@ mod lifecycle_tests { } } + fn with_policy(mut sandbox: DriverSandbox, policy: SandboxPolicy) -> DriverSandbox { + sandbox.spec.as_mut().unwrap().policy = Some(policy); + sandbox + } + fn ready_condition(sb: &DriverSandbox) -> Option { sb.status .as_ref()? @@ -1034,11 +1032,8 @@ mod lifecycle_tests { let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); let policy = fs_policy(&[&share]); - let sb = driver_sandbox_with_command("sb-pos", &share, cmd); - backend - .create_sandbox(&sb, Some(&policy)) - .await - .expect("create accepted"); + let sb = with_policy(driver_sandbox_with_command("sb-pos", &share, cmd), policy); + backend.create_sandbox(&sb).await.expect("create accepted"); // Self-reported Ready=True (no supervisor) once the agent exec launches. let ready = wait_for(&backend, "sb-pos", |s| { @@ -1090,11 +1085,8 @@ mod lifecycle_tests { let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); let policy = fs_policy(&[&share]); - let sb = driver_sandbox_with_command("sb-pc", &share, cmd); - backend - .create_sandbox(&sb, Some(&policy)) - .await - .expect("create accepted"); + let sb = with_policy(driver_sandbox_with_command("sb-pc", &share, cmd), policy); + backend.create_sandbox(&sb).await.expect("create accepted"); let ready = wait_for(&backend, "sb-pc", |s| { ready_condition(s).is_some_and(|c| c.status == "True" && c.reason == "AgentRunning") @@ -1146,11 +1138,9 @@ mod lifecycle_tests { let mut stream = backend.watch_sandboxes().await; let policy = fs_policy(&[&share]); + let sandbox = with_policy(driver_sandbox_with_command("sb-neg", &share, cmd), policy); backend - .create_sandbox( - &driver_sandbox_with_command("sb-neg", &share, cmd), - Some(&policy), - ) + .create_sandbox(&sandbox) .await .expect("create accepted"); @@ -1203,11 +1193,9 @@ mod lifecycle_tests { ]; let backend = MxcComputeBackend::new_mocked(MxcComputeConfig::default()); let policy = fs_policy(&[&share]); + let sandbox = with_policy(driver_sandbox_with_command("sb-stop", "", command), policy); backend - .create_sandbox( - &driver_sandbox_with_command("sb-stop", "", command), - Some(&policy), - ) + .create_sandbox(&sandbox) .await .expect("create accepted"); wait_for(&backend, "sb-stop", |sandbox| { @@ -1247,8 +1235,9 @@ mod lifecycle_tests { binaries: Vec::new(), }, ); + let sandbox = with_policy(driver_sandbox("sb-net"), policy); let error = backend - .create_sandbox(&driver_sandbox("sb-net"), Some(&policy)) + .create_sandbox(&sandbox) .await .expect_err("unmappable policy must fail CreateSandbox synchronously"); assert_eq!(error.code(), tonic::Code::InvalidArgument); diff --git a/crates/openshell-driver-mxc/src/grpc.rs b/crates/openshell-driver-mxc/src/grpc.rs index 17993a01a0..0fa23f7ac9 100644 --- a/crates/openshell-driver-mxc/src/grpc.rs +++ b/crates/openshell-driver-mxc/src/grpc.rs @@ -60,8 +60,7 @@ impl ComputeDriver for ComputeDriverService { let sandbox = request .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.backend - .validate_sandbox_create(&sandbox, request.policy.as_ref())?; + self.backend.validate_sandbox_create(&sandbox)?; Ok(Response::new(ValidateSandboxCreateResponse {})) } @@ -104,9 +103,7 @@ impl ComputeDriver for ComputeDriverService { let sandbox = request .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.backend - .create_sandbox(&sandbox, request.policy.as_ref()) - .await?; + self.backend.create_sandbox(&sandbox).await?; Ok(Response::new(CreateSandboxResponse {})) } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 0bcae6ddfa..aa41017663 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -515,8 +515,7 @@ impl PodmanComputeDriver { default_image: self.config.default_image.clone(), gateway_manages_lifecycle: true, supports_sandbox_authentication: false, - sandbox_runtime_control: - openshell_core::proto::compute::v1::SandboxRuntimeControl::Supervisor.into(), + driver_reports_runtime_readiness: false, }) } diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index cb06e49ed7..222c617c2f 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -508,7 +508,6 @@ mod tests { client .validate_sandbox_create(request_with_traceparent(ValidateSandboxCreateRequest { sandbox: None, - policy: None, })) .await .expect_err("missing sandbox should fail"); diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index cffdd508f6..8adcc79f92 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -525,8 +525,7 @@ impl VmDriver { default_image: self.config.default_image.clone(), gateway_manages_lifecycle: true, supports_sandbox_authentication: false, - sandbox_runtime_control: - openshell_core::proto::compute::v1::SandboxRuntimeControl::Supervisor.into(), + driver_reports_runtime_readiness: false, } } @@ -5781,7 +5780,6 @@ mod tests { client .validate_sandbox_create(request_with_traceparent(ValidateSandboxCreateRequest { sandbox: None, - policy: None, })) .await .is_err() @@ -5790,7 +5788,6 @@ mod tests { client .create_sandbox(request_with_traceparent(CreateSandboxRequest { sandbox: None, - policy: None, })) .await .is_err() @@ -5898,7 +5895,6 @@ mod tests { }; let request = request_with_traceparent(CreateSandboxRequest { sandbox: Some(sandbox), - policy: None, }); let mut client = traced_driver_client(driver.clone()).await; diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 03da29055c..053b08de9c 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -28,8 +28,8 @@ use openshell_core::proto::compute::v1::{ GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GpuResourceRequirements as DriverGpuResourceRequirements, ListSandboxesRequest, ResourceRequirements as DriverSandboxResourceRequirements, - SandboxRuntimeControl, StartSandboxRequest, StopSandboxRequest, ValidateSandboxCreateRequest, - WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, + StartSandboxRequest, StopSandboxRequest, ValidateSandboxCreateRequest, WatchSandboxesEvent, + WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, watch_sandboxes_event, }; @@ -263,17 +263,8 @@ pub struct ComputeDriverInfoSnapshot { pub gateway_manages_lifecycle: bool, /// Whether the driver authenticates driver-native sandbox credentials. pub supports_sandbox_authentication: bool, - /// Component responsible for runtime readiness and policy enforcement. - pub sandbox_runtime_control: SandboxRuntimeControl, -} - -fn normalize_sandbox_runtime_control(value: i32) -> SandboxRuntimeControl { - match SandboxRuntimeControl::try_from(value) { - Ok(SandboxRuntimeControl::Driver) => SandboxRuntimeControl::Driver, - Ok(SandboxRuntimeControl::Unspecified | SandboxRuntimeControl::Supervisor) | Err(_) => { - SandboxRuntimeControl::Supervisor - } - } + /// Whether the driver reports runtime readiness without a supervisor session. + pub driver_reports_runtime_readiness: bool, } /// Interval between store-vs-backend reconciliation sweeps. @@ -621,9 +612,7 @@ impl ComputeRuntime { driver_version: capabilities.driver_version, gateway_manages_lifecycle: capabilities.gateway_manages_lifecycle, supports_sandbox_authentication: capabilities.supports_sandbox_authentication, - sandbox_runtime_control: normalize_sandbox_runtime_control( - capabilities.sandbox_runtime_control, - ), + driver_reports_runtime_readiness: capabilities.driver_reports_runtime_readiness, }; let default_image = capabilities.default_image; let gateway_listener_requirements = match driver @@ -764,11 +753,6 @@ impl ComputeRuntime { self.driver_info.supports_sandbox_authentication } - #[must_use] - pub(crate) fn sandbox_runtime_control(&self) -> SandboxRuntimeControl { - self.driver_info.sandbox_runtime_control - } - pub(crate) async fn authenticate_sandbox(&self, credential: &str) -> Result { if !self.supports_sandbox_authentication() { return Err(Status::unimplemented( @@ -842,7 +826,6 @@ impl ComputeRuntime { pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { let driver_sandbox = driver_sandbox_from_public(sandbox, &self.driver_info.name) .map_err(|status| *status)?; - let policy = sandbox.spec.as_ref().and_then(|spec| spec.policy.clone()); self.driver .call( "driver.validate_sandbox_create", @@ -851,7 +834,6 @@ impl ComputeRuntime { driver .validate_sandbox_create(Request::new(ValidateSandboxCreateRequest { sandbox: Some(driver_sandbox), - policy, })) .await }, @@ -869,7 +851,6 @@ impl ComputeRuntime { let sandbox_id = sandbox.object_id().to_string(); let mut driver_sandbox = driver_sandbox_from_public(&sandbox, &self.driver_info.name) .map_err(|status| *status)?; - let policy = sandbox.spec.as_ref().and_then(|spec| spec.policy.clone()); // Create with MustCreate condition to prevent duplicate creation race self.sandbox_index.update_from_sandbox(&sandbox); @@ -926,7 +907,6 @@ impl ComputeRuntime { driver .create_sandbox(Request::new(CreateSandboxRequest { sandbox: Some(driver_sandbox), - policy, })) .await }, @@ -1347,7 +1327,7 @@ impl ComputeRuntime { sandbox, snapshot, session_connected, - self.driver_info.sandbox_runtime_control, + self.driver_info.driver_reports_runtime_readiness, ); }) .await @@ -1801,7 +1781,7 @@ impl ComputeRuntime { sandbox, &snapshot, session_connected, - self.driver_info.sandbox_runtime_control, + self.driver_info.driver_reports_runtime_readiness, ); }, ) @@ -2015,7 +1995,7 @@ impl ComputeRuntime { let watch_runtime = runtime.clone(); let watch_shutdown = shutdown_rx.clone(); tokio::spawn(async move { - watch_runtime.watch_loop(watch_shutdown).await; + Box::pin(watch_runtime.watch_loop(watch_shutdown)).await; }); tokio::spawn(async move { runtime.reconcile_loop(shutdown_rx).await; @@ -2541,7 +2521,7 @@ impl ComputeRuntime { let runtime = self.clone(); let watch_cancel = cancel_rx.clone(); let watch_handle = tokio::spawn(async move { - runtime.watch_loop(watch_cancel).await; + Box::pin(runtime.watch_loop(watch_cancel)).await; }); let runtime = self.clone(); @@ -2848,7 +2828,7 @@ impl ComputeRuntime { sandbox, &incoming, session_connected, - self.driver_info.sandbox_runtime_control, + self.driver_info.driver_reports_runtime_readiness, ); }, ) @@ -3758,6 +3738,7 @@ fn driver_sandbox_spec_from_public( .as_ref() .map(|template| driver_sandbox_template_from_public(template, driver_name)) .transpose()?, + policy: spec.policy.clone(), resource_requirements: spec.resource_requirements.as_ref().map(|requirements| { DriverSandboxResourceRequirements { gpu: requirements @@ -4038,7 +4019,7 @@ fn apply_driver_snapshot( sandbox: &mut Sandbox, incoming: &DriverSandbox, session_connected: bool, - sandbox_runtime_control: SandboxRuntimeControl, + driver_reports_runtime_readiness: bool, ) { let old_phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); let sandbox_name = &incoming.name; @@ -4074,8 +4055,11 @@ fn apply_driver_snapshot( (phase, status) }, |incoming_status| { - let composed = - ComposedPhase::new(incoming_status, session_connected, sandbox_runtime_control); + let composed = ComposedPhase::new( + incoming_status, + session_connected, + driver_reports_runtime_readiness, + ); let mut status = Some(public_status_from_driver( incoming_status, composed.phase, @@ -4229,7 +4213,7 @@ impl ComposedPhase { fn new( incoming_status: &DriverSandboxStatus, session_connected: bool, - sandbox_runtime_control: SandboxRuntimeControl, + driver_reports_runtime_readiness: bool, ) -> Self { let backend_phase = derive_phase(Some(incoming_status)); // A live supervisor session is a stronger readiness signal than the backend phase. @@ -4238,16 +4222,14 @@ impl ComposedPhase { // backend phase overwrite it. let phase = match backend_phase { SandboxPhase::Error | SandboxPhase::Deleting | SandboxPhase::Stopped => backend_phase, - SandboxPhase::Ready if sandbox_runtime_control == SandboxRuntimeControl::Driver => { - SandboxPhase::Ready - } + SandboxPhase::Ready if driver_reports_runtime_readiness => SandboxPhase::Ready, _ if session_connected => SandboxPhase::Ready, _ => SandboxPhase::Provisioning, }; Self { phase, session_connected, - backend_ready_without_session: sandbox_runtime_control != SandboxRuntimeControl::Driver + backend_ready_without_session: !driver_reports_runtime_readiness && backend_phase == SandboxPhase::Ready && !session_connected, } @@ -4532,7 +4514,7 @@ impl ComputeDriver for NoopTestDriver { default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: self.sandbox_authentication.is_some(), - sandbox_runtime_control: SandboxRuntimeControl::Supervisor.into(), + driver_reports_runtime_readiness: false, }, )) } @@ -4676,7 +4658,7 @@ pub async fn new_test_runtime_with_driver( driver_version: "test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication, - sandbox_runtime_control: SandboxRuntimeControl::Supervisor, + driver_reports_runtime_readiness: false, }, telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process: None, @@ -4854,7 +4836,7 @@ mod tests { default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: false, - sandbox_runtime_control: SandboxRuntimeControl::Supervisor.into(), + driver_reports_runtime_readiness: false, })) } @@ -5195,7 +5177,7 @@ mod tests { default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: false, - sandbox_runtime_control: SandboxRuntimeControl::Supervisor.into(), + driver_reports_runtime_readiness: false, })) } @@ -5405,7 +5387,7 @@ mod tests { driver_version: "test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: false, - sandbox_runtime_control: SandboxRuntimeControl::Supervisor, + driver_reports_runtime_readiness: false, }, telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process: None, @@ -6037,7 +6019,7 @@ mod tests { ) -> (watch::Sender, tokio::task::JoinHandle<()>) { let (shutdown_tx, shutdown_rx) = watch::channel(false); let runtime = Arc::new(runtime.clone()); - let handle = tokio::spawn(async move { runtime.watch_loop(shutdown_rx).await }); + let handle = tokio::spawn(async move { Box::pin(runtime.watch_loop(shutdown_rx)).await }); tokio::time::timeout(Duration::from_secs(1), driver.watch_started.notified()) .await .expect("watch loop did not start"); @@ -6062,19 +6044,7 @@ mod tests { } #[test] - fn unspecified_runtime_control_preserves_supervisor_compatibility() { - assert_eq!( - normalize_sandbox_runtime_control(SandboxRuntimeControl::Unspecified.into()), - SandboxRuntimeControl::Supervisor - ); - assert_eq!( - normalize_sandbox_runtime_control(i32::MAX), - SandboxRuntimeControl::Supervisor - ); - } - - #[test] - fn driver_controlled_runtime_uses_driver_readiness() { + fn driver_reported_runtime_uses_driver_readiness() { let status = make_driver_status(DriverCondition { r#type: "Ready".to_string(), status: "True".to_string(), @@ -6083,7 +6053,7 @@ mod tests { last_transition_time: String::new(), }); - let composed = ComposedPhase::new(&status, false, SandboxRuntimeControl::Driver); + let composed = ComposedPhase::new(&status, false, true); assert_eq!(composed.phase, SandboxPhase::Ready); assert!(!composed.backend_ready_without_session); @@ -10394,14 +10364,12 @@ mod tests { remote .validate_sandbox_create(Request::new(ValidateSandboxCreateRequest { sandbox: Some(sandbox.clone()), - policy: None, })) .await .unwrap(); remote .create_sandbox(Request::new(CreateSandboxRequest { sandbox: Some(sandbox.clone()), - policy: None, })) .await .unwrap(); @@ -10573,11 +10541,7 @@ mod tests { let validated = match &calls[2] { FakeComputeDriverCall::ValidateSandboxCreate { sandbox: Some(sandbox), - policy: Some(policy), - } => { - assert_eq!(policy.version, 42); - sandbox - } + } => sandbox, other => panic!("expected ValidateSandboxCreate call, got {other:?}"), }; let driver_config = validated @@ -10588,12 +10552,19 @@ mod tests { .expect("selected driver_config should be forwarded"); assert!(driver_config.fields.contains_key("pool")); assert!(!driver_config.fields.contains_key("network_mode")); + assert_eq!( + validated + .spec + .as_ref() + .and_then(|spec| spec.policy.as_ref()) + .map(|policy| policy.version), + Some(42) + ); assert!(matches!( &calls[3], - FakeComputeDriverCall::CreateSandbox { - policy: Some(policy), - .. - } if policy.version == 42 + FakeComputeDriverCall::CreateSandbox { sandbox: Some(sandbox) } + if sandbox.spec.as_ref().and_then(|spec| spec.policy.as_ref()) + .is_some_and(|policy| policy.version == 42) )); driver.clear_calls(); diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index fc140fec71..0e4c74af95 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -23,7 +23,6 @@ use crate::provider_profile_sources::EffectiveProviderProfileCatalog; #[cfg(test)] use crate::provider_profile_sources::ProviderProfileSources; use openshell_core::net::{is_always_blocked_ip, is_internal_ip}; -use openshell_core::proto::compute::v1::SandboxRuntimeControl; use openshell_core::proto::policy_merge_operation; use openshell_core::proto::setting_value; use openshell_core::proto::{ @@ -3136,19 +3135,6 @@ pub(super) async fn handle_get_sandbox_provider_environment( // Update config handler (policy + settings mutations) // --------------------------------------------------------------------------- -fn validate_live_policy_update_support( - sandbox_runtime_control: SandboxRuntimeControl, - has_policy: bool, - has_merge_ops: bool, -) -> Result<(), Status> { - if (has_policy || has_merge_ops) && sandbox_runtime_control == SandboxRuntimeControl::Driver { - return Err(Status::failed_precondition( - "live policy updates are not supported when the selected compute driver owns runtime policy enforcement; recreate the sandbox to apply the new policy", - )); - } - Ok(()) -} - pub(super) async fn handle_update_config( state: &Arc, request: Request, @@ -3223,11 +3209,6 @@ async fn handle_update_config_inner( "one of policy, setting_key, or merge_operations must be provided", )); } - validate_live_policy_update_support( - state.compute.sandbox_runtime_control(), - has_policy, - has_merge_ops, - )?; if req.global { if !req.annotations.is_empty() { return Err(Status::invalid_argument( @@ -6915,26 +6896,6 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use tonic::Code; - #[test] - fn driver_controlled_runtime_rejects_sandbox_policy_replacement_and_merge_updates() { - for (has_policy, has_merge_ops) in [(true, false), (false, true)] { - let error = validate_live_policy_update_support( - SandboxRuntimeControl::Driver, - has_policy, - has_merge_ops, - ) - .expect_err("driver-controlled runtimes must reject policy mutations after launch"); - assert_eq!(error.code(), Code::FailedPrecondition); - } - - let error = validate_live_policy_update_support(SandboxRuntimeControl::Driver, true, false) - .expect_err("global policy replacement also changes desired state for live sandboxes"); - assert_eq!(error.code(), Code::FailedPrecondition); - - validate_live_policy_update_support(SandboxRuntimeControl::Supervisor, true, false) - .expect("supervisor-controlled runtimes support live policy replacement"); - } - /// Wrap a request with a user `Principal` so handler scope guards treat /// the test caller as a CLI user. Most handler tests exercise /// user-facing behavior and should not trip sandbox equality checks. diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index 6b786699c4..0e4c318192 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -4,7 +4,6 @@ //! Test fixtures for exercising gateway integration points. use futures::{Stream, stream}; -use openshell_core::proto::SandboxPolicy; #[cfg(unix)] use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_core::proto::compute::v1::{ @@ -41,7 +40,6 @@ pub enum FakeComputeDriverCall { GetGatewayListenerRequirements, ValidateSandboxCreate { sandbox: Option, - policy: Option, }, GetSandbox { sandbox_id: String, @@ -50,7 +48,6 @@ pub enum FakeComputeDriverCall { ListSandboxes, CreateSandbox { sandbox: Option, - policy: Option, }, StopSandbox { sandbox_id: String, @@ -99,8 +96,7 @@ impl FakeComputeDriver { default_image: "openshell/sandbox:test".to_string(), gateway_manages_lifecycle: false, supports_sandbox_authentication: false, - sandbox_runtime_control: - openshell_core::proto::compute::v1::SandboxRuntimeControl::Supervisor.into(), + driver_reports_runtime_readiness: false, }, gateway_listener_requirements: Vec::new(), gateway_listener_requirements_supported: true, @@ -295,7 +291,6 @@ impl ComputeDriver for FakeComputeDriver { .calls .push(FakeComputeDriverCall::ValidateSandboxCreate { sandbox: request.sandbox, - policy: request.policy, }); }); Ok(Response::new(ValidateSandboxCreateResponse {})) @@ -351,10 +346,9 @@ impl ComputeDriver for FakeComputeDriver { if let Some(sandbox) = sandbox.as_ref() { state.sandboxes.insert(sandbox.id.clone(), sandbox.clone()); } - state.calls.push(FakeComputeDriverCall::CreateSandbox { - sandbox, - policy: request.policy, - }); + state + .calls + .push(FakeComputeDriverCall::CreateSandbox { sandbox }); }); Ok(Response::new(CreateSandboxResponse {})) } diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 0680d208f3..f00c49afd6 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -13,10 +13,11 @@ The gateway's configured compute driver determines how OpenShell creates each sa Most compute drivers run the OpenShell supervisor inside the sandbox workload. The supervisor launches the agent process, applies policy, routes egress through the proxy, injects configured credentials, and maintains the gateway session. -A driver may instead advertise driver-owned runtime control. In that mode, the -gateway sends the canonical policy in the validation and creation requests, the -driver applies it before launch, and driver-reported readiness does not require -a supervisor session. The Windows MXC driver uses this mode. +A driver may instead set `driver_reports_runtime_readiness`. In that mode, +driver-reported readiness does not require a supervisor session. The canonical +create-time policy is part of `DriverSandboxSpec`; a driver that enforces policy +outside the standard supervisor fetches later revisions through the existing +sandbox configuration API. The Windows MXC driver reports its own readiness. Stop stops compute but retains the sandbox record and the driver's persistent workspace boundary. Start reactivates the same driver resource. diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index e33834b824..b737e7de62 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -72,17 +72,6 @@ service ComputeDriver { message GetCapabilitiesRequest {} -// Component responsible for the sandbox runtime control plane. -enum SandboxRuntimeControl { - // Compatibility default for drivers that predate this capability. The - // gateway treats this as SANDBOX_RUNTIME_CONTROL_SUPERVISOR. - SANDBOX_RUNTIME_CONTROL_UNSPECIFIED = 0; - // The standard OpenShell supervisor owns readiness and live policy updates. - SANDBOX_RUNTIME_CONTROL_SUPERVISOR = 1; - // The compute driver owns readiness and applies policy while provisioning. - SANDBOX_RUNTIME_CONTROL_DRIVER = 2; -} - message GetCapabilitiesResponse { reserved 4, 5; reserved "supports_gpu", "gpu_count"; @@ -98,8 +87,10 @@ message GetCapabilitiesResponse { bool gateway_manages_lifecycle = 6; // Whether AuthenticateSandbox is implemented by this driver. bool supports_sandbox_authentication = 7; - // Component responsible for runtime readiness and policy enforcement. - SandboxRuntimeControl sandbox_runtime_control = 8; + // Whether the driver reports runtime readiness itself. When false, the + // gateway waits for the standard OpenShell supervisor session in addition + // to the driver's platform-ready observation. + bool driver_reports_runtime_readiness = 8; } message AuthenticateSandboxRequest { @@ -168,6 +159,10 @@ message DriverSandboxSpec { map environment = 5; // Runtime template consumed by the driver during provisioning. DriverSandboxTemplate template = 6; + // Canonical effective policy supplied to the driver for validation and + // provisioning. Drivers that enforce policy outside the standard supervisor + // use GetSandboxConfig to fetch later revisions. + openshell.sandbox.v1.SandboxPolicy policy = 7; // Portable resource requirements used by the gateway for driver selection // and by drivers for provisioning. ResourceRequirements resource_requirements = 9; @@ -306,9 +301,6 @@ message DriverPlatformEvent { message ValidateSandboxCreateRequest { // Proposed sandbox configuration to validate before provisioning. DriverSandbox sandbox = 1; - // Canonical policy the selected driver must validate when it owns runtime - // control. Supervisor-controlled drivers may ignore this field. - openshell.sandbox.v1.SandboxPolicy policy = 2; } message ValidateSandboxCreateResponse {} @@ -335,9 +327,6 @@ message ListSandboxesResponse { message CreateSandboxRequest { // Sandbox configuration to provision on the compute platform. DriverSandbox sandbox = 1; - // Canonical policy the selected driver must apply when it owns runtime - // control. Supervisor-controlled drivers may ignore this field. - openshell.sandbox.v1.SandboxPolicy policy = 2; } message CreateSandboxResponse {}