diff --git a/Cargo.lock b/Cargo.lock index ac8d20e2d4..2a614f6015 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4176,8 +4176,17 @@ name = "openshell-isolation-interface" version = "0.0.0" dependencies = [ "async-trait", + "libc", "openshell-core", + "rcgen", + "rustls 0.23.38", + "rustls-pemfile", + "serde", + "serde_json", + "socket2 0.6.3", + "thiserror 2.0.18", "tokio", + "tokio-rustls 0.26.4", ] [[package]] @@ -4281,12 +4290,15 @@ dependencies = [ name = "openshell-sandbox" version = "0.0.0" dependencies = [ + "base64 0.22.1", "clap", "futures", + "libc", "miette", "nix 0.29.0", "openshell-core", "openshell-extension-core", + "openshell-isolation-interface", "openshell-ocsf", "openshell-policy", "openshell-supervisor-middleware", @@ -4295,12 +4307,15 @@ dependencies = [ "openshell-supervisor-process", "prost", "prost-types", + "rcgen", "rustls", + "rustls-pemfile", "serde", "serde_json", "temp-env", "tempfile", "tokio", + "tokio-rustls 0.26.4", "tokio-tungstenite 0.26.2", "tonic", "tracing", @@ -5564,6 +5579,7 @@ dependencies = [ "ring", "rustls-pki-types", "time", + "x509-parser", "yasna", ] @@ -8650,6 +8666,7 @@ dependencies = [ "lazy_static", "nom", "oid-registry", + "ring", "rusticata-macros", "thiserror 1.0.69", "time", diff --git a/Cargo.toml b/Cargo.toml index 6936439a3b..16845307a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ http-body-util = "0.1" tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } rustls = { version = "0.23", default-features = false, features = ["std", "logging", "tls12", "ring"] } rustls-pemfile = "2" -rcgen = { version = "0.13", features = ["crypto", "pem"] } +rcgen = { version = "0.13", features = ["crypto", "pem", "x509-parser"] } webpki-roots = "1" rustls-native-certs = "0.8" diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e1e731a0ce..f81563a4bf 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -1,10 +1,11 @@ # Compute Runtimes Compute runtimes create, stop, start, delete, and watch sandbox workloads for the -gateway. Supervisor-controlled runtimes start a workload that runs the -`openshell-sandbox` supervisor, which enforces the sandbox contract locally. -Driver-controlled runtimes apply the canonical sandbox policy while -provisioning and report workload readiness directly. +gateway. They do not replace sandbox policy enforcement. A runtime either starts +the co-located `openshell-sandbox` supervisor or provisions its RFC 0012 +`--mode=control` and `--mode=boundary` placements. Runtimes without the standard +supervisor apply the canonical sandbox policy while provisioning and report +workload readiness directly. ## Driver Contract @@ -12,9 +13,14 @@ Each runtime receives a sandbox spec and canonical policy from the gateway and is responsible for: - Selecting the sandbox image. -- For supervisor-controlled runtimes, injecting sandbox identity and gateway - callback configuration, supplying callback credentials, and providing the - supervisor binary or image. +- Injecting sandbox identity and gateway callback configuration. +- Supplying TLS or secret material for supervisor callbacks. +- Providing the supervisor binary or image in the workload. +- Provisioning protected control and boundary configs plus a private Unix socket, + TLS-authenticated TCP, or vsock transport when the supervisor is separated. + Runtime-specific code supplies immutable resource claims and transport + coordinates; the shared boundary protocol supplies lifecycle, exec, signaling, + forwarding, and binary identity semantics. - For runtimes without the standard supervisor, validating and applying the canonical policy before launching the workload. - Forwarding the exact canonical main-process argv and TTY mode without shell @@ -247,10 +253,10 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| -| Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | +| Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Creates the workload with Docker `network_mode=none`; a host control process mediates egress and access over a private bind-mounted Unix socket. | | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | -| Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | -| VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | +| Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | The proxy-pod topology fences workload egress with NetworkPolicy, runs boundary mode as the workload entrypoint, and runs control mode in a separate zero-capability pod over per-boundary TLS. It requires an enforcing CNI and trusted sandbox namespace. | +| VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | The guest is NIC-less and runs boundary mode as PID 1; host control owns gateway networking and reaches the guest over vsock. The driver persists each accepted launch request and writable overlay for restart. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 055ef7e4a3..f498e5d552 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -1,8 +1,7 @@ # Sandbox -A sandbox is the runtime boundary where agent code executes. It is created by a -compute runtime and managed inside the workload by `openshell-sandbox`, the -sandbox supervisor. +A sandbox is the runtime boundary where agent code executes. A compute runtime +creates it, and `openshell-sandbox` operates it as one logical supervisor. ## Runtime Model @@ -21,20 +20,52 @@ container-granted capabilities. This is fail-closed: the supervisor retains aborts unless the bounding set ends up empty. A `setpcap` `EPERM` is tolerated only when the set is already empty; any other outcome fails the spawn. +Separated runtimes split the same logical supervisor into two placements: + +| Mode | Placement and authority | +|---|---| +| `openshell-sandbox --mode=control` | Runs outside the workload boundary. Owns gateway credentials, the admitted policy, RFC 0012 lifecycle, network-policy decisions, SSH, and gateway relays. | +| `openshell-sandbox --mode=boundary --boundary-config ` | Runs inside or adjacent to the workload. Owns process launch and observation, exec signaling, PTY control, loopback forwarding, egress capture, and binary identity. It has no gateway credentials or independent policy authority. | + +The compute driver provisions the two protected configurations and their private +Unix-socket, TLS-authenticated TCP, or vsock transport. TCP across a shared or +operator-managed network uses a driver-provisioned trust root and verified server +name; NetworkPolicy alone is not a confidentiality boundary. All configurations carry the +same boundary ID, bootstrap credential, protocol version, and immutable driver resource claims. +The shared remote backend authenticates each request and rejects a resource-claim +mismatch. Driver crates do not appear in generic supervisor lifecycle, network, +SSH, or session code. + +Control mode can expose semantic TCP readiness after the boundary reaches +`Running`. The bind IP is explicit and address-family neutral; Kubernetes +drivers inject the pod IP so IPv4 and IPv6 probes reach the same listener. + +For cross-UID Unix-socket placements, the driver-owned parent directory limits +socket reachability and the bootstrap credential authenticates every request. +Boundary mode can re-own a bind-mounted config as root-only before starting the +workload when its host ownership could otherwise coincide with the workload UID. + ## Startup Flow 1. The compute runtime starts the workload with sandbox identity, callback endpoint, TLS or secret material, image metadata, and initial command. -2. The supervisor loads policy and runtime settings from local files or the +2. Control mode loads policy and runtime settings from local files or the gateway, depending on mode. -3. It prepares filesystem access, process restrictions, network namespace - routing, trust stores, provider credential resolution, and inference routes. -4. It launches the persisted canonical main-process argv and retains its PTY - or pipes in the main-session multiplexer. -5. It starts the policy proxy and local SSH server. -6. It opens a supervisor session back to the gateway for connect, exec, file +3. The isolation backend attaches and confirms standing enforcement. Boundary + mode prepares boundary-local filesystem, process, and egress controls. +4. Control mode connects network mediation, then confirms enforcement and starts + the admitted main process through the backend. Boundary + mode applies launch-time controls before its first untrusted instruction. +5. Control mode starts the local SSH server. Exec and loopback + streams cross the shared boundary protocol when the modes are separated. +6. Control mode opens a supervisor session back to the gateway for connect, exec, file sync, config polling, and log push. +When the admitted main process exits, its status and retained terminal output +remain available. The confirmed boundary and control-owned access plane continue +to serve policy-authorized exec and loopback forwarding until explicit stop or +delete tears down the boundary and terminates any remaining workload processes. + ## Isolation Layers OpenShell uses overlapping controls rather than a single sandbox primitive: @@ -109,6 +140,15 @@ generation-pinned authorization form the transparent TCP security boundary. Docker and Podman do not currently advertise usable IPv6 egress for this substrate, so AAAA queries return NOERROR/NODATA and IPv6 DNS remains fenced. +Network-rule ownership depends on supervisor placement. In legacy combined +mode, `openshell-sandbox` creates the workload network namespace and owns its +`ip`, iptables, and nftables setup and cleanup. In delegated control/boundary +mode, the compute driver provisions the protected network substrate before +control attaches; boundary mode owns only boundary-local observation and the +listener handed to it. Control mode evaluates policy but must not mutate a +driver-owned namespace or assume that legacy `ip`/nft cleanup applies. Drivers +must tear down their own rules and namespace resources with the workload. + Provider credential placeholders are resolved through the live provider state for each HTTP request, after destination and L7 policy admission. A static credential resolves only when the request host, port, and path match an endpoint @@ -527,7 +567,11 @@ engine with a gateway policy revision. to new connections or the next parsed HTTP request where the proxy can safely re-evaluate. - If the supervisor relay drops, the sandbox can keep running, but connect and - exec operations fail until the supervisor registers again. + exec operations fail until the supervisor registers again. In a separated + topology, a replacement control process replays the identical boundary + lifecycle and receives the existing process handle. The boundary rejects + changed launch inputs and releases the single main-process attachment when + the old control transport closes. - If the canonical main process exits, the supervisor durably reports the normalized result immediately. A foreground create declares a one-shot main attachment, so the supervisor accepts it even after a fast process exits, diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index 2b1537a21b..d9055fa319 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -359,6 +359,17 @@ impl ProviderCredentialState { /// here so SDKs can read them at startup. /// 3. Everything else stays as placeholders for proxy-time resolution. pub fn child_env_with_gcp_resolved(&self) -> HashMap { + self.child_env_snapshot_with_gcp_resolved().1 + } + + /// Return the current revision and its workload-facing environment from + /// one state snapshot. + /// + /// Remote isolation boundaries use the pair as a revisioned update. The + /// revision must describe the exact environment sent across the boundary, + /// so callers must not obtain the two values through separate lock + /// acquisitions. + pub fn child_env_snapshot_with_gcp_resolved(&self) -> (u64, HashMap) { use crate::google_cloud; let inner = self @@ -376,7 +387,7 @@ impl ProviderCredentialState { .any(|key| env.contains_key(*key) && inner.non_secret_environment_keys.contains(*key)); if !has_gcp_metadata && !has_gcp_config { - return env; + return (inner.current.revision, env); } if has_gcp_metadata { @@ -414,7 +425,44 @@ impl ProviderCredentialState { } } - env + (inner.current.revision, env) + } + + /// Compare and install a workload-facing environment snapshot. + /// + /// Provider environment revisions are opaque content identities, not + /// ordered counters. The expected revision makes retries idempotent while + /// rejecting updates based on a stale view of the boundary state. + pub fn compare_and_install_child_env_snapshot( + &self, + expected_revision: u64, + revision: u64, + mut child_env: HashMap, + ) -> u64 { + let mut inner = self + .inner + .write() + .expect("provider credential state poisoned"); + if revision == inner.current.revision || expected_revision != inner.current.revision { + return inner.current.revision; + } + + for key in &inner.suppressed_keys { + child_env.remove(key); + } + inner.current = Arc::new(ProviderCredentialSnapshot { + revision, + child_env, + dynamic_credentials: HashMap::new(), + }); + inner.generations.clear(); + inner.current_resolver = None; + inner.combined_resolver = None; + inner.non_secret_environment_keys.clear(); + inner.static_credential_bindings.clear(); + inner.known_static_credential_keys.clear(); + inner.static_credential_identity_epochs.clear(); + revision } /// Return the GCP token placeholder and its remaining lifetime in seconds. @@ -2117,6 +2165,40 @@ mod tests { ); } + #[test] + fn child_env_snapshot_update_uses_opaque_revision_cas() { + let state = ProviderCredentialState::from_child_env_snapshot( + 4, + HashMap::from([("TOKEN".to_string(), "four".to_string())]), + ); + + assert_eq!( + state.compare_and_install_child_env_snapshot( + 4, + 6, + HashMap::from([("TOKEN".to_string(), "six".to_string())]), + ), + 6 + ); + assert_eq!( + state.compare_and_install_child_env_snapshot( + 4, + 5, + HashMap::from([("TOKEN".to_string(), "stale".to_string())]), + ), + 6 + ); + assert_eq!( + state.compare_and_install_child_env_snapshot(6, 2, HashMap::new()), + 2, + "opaque revisions may move numerically backwards" + ); + + let (revision, env) = state.child_env_snapshot_with_gcp_resolved(); + assert_eq!(revision, 2); + assert!(env.is_empty(), "an empty snapshot must revoke the old env"); + } + #[test] fn stale_generation_falls_back_to_current_credential_after_retention_window() { let state = ProviderCredentialState::from_environment( diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 99ac55fe6e..f76258ba04 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -26,6 +26,9 @@ pub const SSH_SOCKET_PATH: &str = "OPENSHELL_SSH_SOCKET_PATH"; /// Log level for the sandbox supervisor (e.g. `"debug"`, `"info"`, `"warn"`). pub const LOG_LEVEL: &str = "OPENSHELL_LOG_LEVEL"; +/// IP address for the control-mode semantic readiness listener. +pub const HEALTH_BIND_IP: &str = "OPENSHELL_HEALTH_BIND_IP"; + /// Versioned specification for the exact canonical main process. /// /// Most drivers use JSON directly. Transports that cannot preserve spaces in @@ -122,6 +125,13 @@ pub const TELEMETRY_ENABLED: &str = "OPENSHELL_TELEMETRY_ENABLED"; /// `"sidecar"`; the default combined supervisor path omits it. pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY"; +/// The isolation backend admitted by the deployment configuration (RFC 0012). +/// +/// Delivered on a channel separate from the topology descriptor so descriptor +/// verification against the admitted backend is not self-referential. Required +/// whenever a topology descriptor is supplied. +pub const ADMITTED_ISOLATION_BACKEND: &str = "OPENSHELL_ADMITTED_ISOLATION_BACKEND"; + /// Network enforcement backend selected by the compute driver. pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE"; @@ -154,6 +164,17 @@ pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; /// by workload child processes. pub const PROXY_TLS_DIR: &str = "OPENSHELL_PROXY_TLS_DIR"; +/// Optional path to a durable PEM-encoded interception CA certificate. +/// Must be configured together with [`PROXY_CA_KEY`]. +pub const PROXY_CA_CERT: &str = "OPENSHELL_PROXY_CA_CERT"; + +/// Optional path to the private key for [`PROXY_CA_CERT`]. +/// Must be configured together with the certificate path. +pub const PROXY_CA_KEY: &str = "OPENSHELL_PROXY_CA_KEY"; + +/// Whether the control-owned SSH Unix socket is shared across trusted UIDs. +pub const SSH_SOCKET_SHARED: &str = "OPENSHELL_SSH_SOCKET_SHARED"; + /// Path to the CA certificate for mTLS communication with the gateway. pub const TLS_CA: &str = "OPENSHELL_TLS_CA"; diff --git a/crates/openshell-isolation-interface/Cargo.toml b/crates/openshell-isolation-interface/Cargo.toml index 647f19dad4..401eb280cc 100644 --- a/crates/openshell-isolation-interface/Cargo.toml +++ b/crates/openshell-isolation-interface/Cargo.toml @@ -13,9 +13,20 @@ repository.workspace = true [dependencies] openshell-core = { path = "../openshell-core", default-features = false } async-trait = "0.1" +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true } +rustls = { workspace = true } +rustls-pemfile = { workspace = true } +tokio-rustls = { workspace = true } +socket2 = { workspace = true } + +[target.'cfg(unix)'.dependencies] +libc = "0.2" [dev-dependencies] +rcgen = { workspace = true } tokio = { workspace = true } [lints] diff --git a/crates/openshell-isolation-interface/src/boundary_protocol.rs b/crates/openshell-isolation-interface/src/boundary_protocol.rs new file mode 100644 index 0000000000..231106be80 --- /dev/null +++ b/crates/openshell-isolation-interface/src/boundary_protocol.rs @@ -0,0 +1,912 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Versioned control protocol shared by every remote isolation boundary. +//! +//! Drivers choose and provision the transport, but they do not redefine the +//! process lifecycle, streaming, identity, or authentication messages. The +//! control and boundary roles exchange these length-delimited JSON frames over +//! a private Unix socket, authenticated TCP connection, or virtio-vsock stream. + +use std::fmt; +use std::io; +use std::io::{Read, Write}; +use std::path::PathBuf; + +use crate::AgentSpec; +use crate::contract::Sha256Digest; +use crate::contract::{ + BackendError, BinaryIdentity, BoundaryExitStatus, BoundarySignal, ExecSpec, INTERFACE_VERSION, + ResolveError, TopologyDescriptor, +}; +use openshell_core::policy::{ + FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkMode, NetworkPolicy, + ProcessPolicy, ProxyPolicy, SandboxPolicy, +}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +pub const MAX_CONTROL_FRAME_BYTES: usize = 1024 * 1024; +pub const STREAM_STDIN: u8 = 0; +pub const STREAM_STDOUT: u8 = 1; +pub const STREAM_STDERR: u8 = 2; +pub const STREAM_EXIT: u8 = 3; +pub const STREAM_STDIN_CLOSED: u8 = 4; +pub const MAX_STREAM_FRAME_BYTES: usize = 64 * 1024; +/// Version of the driver-to-supervisor boundary descriptor and config shape. +pub const BOUNDARY_PROTOCOL_VERSION: u32 = 3; + +/// Control-side endpoint for a driver-provisioned boundary. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoundaryTransport { + /// Private Unix socket, including a host endpoint mapped to guest vsock. + Unix { socket_path: PathBuf }, + /// Authenticated TCP endpoint, normally scoped by the runtime network. + Tcp { address: std::net::SocketAddr }, + /// Confidential TCP endpoint authenticated with a driver-provisioned CA. + TlsTcp { + address: std::net::SocketAddr, + /// DNS identity expected in the boundary's server certificate. + server_name: String, + /// PEM-encoded CA certificate used only for this boundary. + ca_certificate_pem: String, + }, + /// Linux host `AF_VSOCK` connection to a guest boundary. + Vsock { guest_cid: u32, control_port: u32 }, +} + +/// Boundary-side listener provisioned by a compute driver. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoundaryListener { + /// Private Unix socket shared with a host or sidecar transport adapter. + Unix { socket_path: PathBuf }, + /// TCP listener address. An unspecified IP is valid for the boundary-side + /// bind; the control-side transport must contain a concrete dial target. + Tcp { address: std::net::SocketAddr }, + /// TLS-protected TCP listener. Drivers provision these files inside the + /// boundary; the private key is never included in the control topology. + TlsTcp { + address: std::net::SocketAddr, + certificate_chain_path: PathBuf, + private_key_path: PathBuf, + }, + /// Guest `AF_VSOCK` listener. + Vsock { control_port: u32 }, +} + +/// Workload identity selected by the compute driver and resolved at the +/// boundary before the first workload instruction executes. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoundaryAgentIdentity { + /// A platform-selected numeric identity, as used by VM and Kubernetes. + Resolved { uid: u32, gid: u32 }, + /// The image's raw OCI `Config.User` declaration, as used by Docker. + OciUser { declaration: String }, + /// No driver-authoritative identity metadata is available. + None, +} + +/// Protected control descriptor delivered to `--mode=control`. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryTopology { + /// Wire-format version. Must match [`BOUNDARY_PROTOCOL_VERSION`]. + pub protocol_version: u32, + /// Stable identity of the boundary, normally the sandbox ID. + pub boundary_id: String, + /// Driver-provisioned control endpoint. + pub transport: BoundaryTransport, + /// Trusted dial target for well-known host-gateway aliases, when the + /// network supervisor cannot use the boundary's resolver view. + #[serde(default)] + pub host_gateway_ip: Option, + /// Driver-specific immutable resource coordinates bound at attach (for + /// example pod UID, VM generation, or container ID). + #[serde(default)] + pub resource_claims: std::collections::BTreeMap, + /// Per-boundary authentication secret; never exposed to workload code. + pub bootstrap_token: String, +} + +impl fmt::Debug for BoundaryTopology { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BoundaryTopology") + .field("protocol_version", &self.protocol_version) + .field("boundary_id", &self.boundary_id) + .field("transport", &self.transport) + .field("host_gateway_ip", &self.host_gateway_ip) + .field("resource_claims", &self.resource_claims) + .field("bootstrap_token", &"") + .finish() + } +} + +impl BoundaryTopology { + /// Encode this topology as the shared RFC 0012 descriptor admitted for + /// `backend_name`. + pub fn descriptor( + &self, + backend_name: impl Into, + ) -> Result { + let payload = serde_json::to_vec(self) + .map_err(|error| BackendError::Descriptor(format!("encode topology: {error}")))?; + Ok(TopologyDescriptor { + version: INTERFACE_VERSION, + backend_name: backend_name.into(), + payload, + }) + } +} + +/// Protected configuration delivered to `--mode=boundary`. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryConfig { + /// Wire-format version. Must match [`BOUNDARY_PROTOCOL_VERSION`]. + pub protocol_version: u32, + /// Stable identity expected in every authenticated request. + pub boundary_id: String, + /// Per-boundary authentication secret. + pub bootstrap_token: String, + /// Re-own this config as boundary-root-only after it has been decoded and + /// before any workload process can start. Drivers use this when the + /// protected delivery file's host owner may match the workload UID. + #[serde(default)] + pub protect_config_file: bool, + /// Driver-provisioned listener. + pub listener: BoundaryListener, + /// Immutable coordinates the boundary requires from the control-side + /// topology descriptor before accepting attachment. + #[serde(default)] + pub resource_claims: std::collections::BTreeMap, + /// Driver-selected identity for the admitted workload. + pub agent_identity: BoundaryAgentIdentity, + /// Absolute, driver-owned helper runtime used for namespace setup. + pub trusted_runtime_root: PathBuf, + /// Driver-resolved environment exposed only to workload processes. + #[serde(default)] + pub child_env: std::collections::HashMap, +} + +impl fmt::Debug for BoundaryConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BoundaryConfig") + .field("protocol_version", &self.protocol_version) + .field("boundary_id", &self.boundary_id) + .field("bootstrap_token", &"") + .field("protect_config_file", &self.protect_config_file) + .field("listener", &self.listener) + .field("resource_claims", &self.resource_claims) + .field("agent_identity", &self.agent_identity) + .field("trusted_runtime_root", &self.trusted_runtime_root) + .field("child_env_keys", &self.child_env.keys().collect::>()) + .finish() + } +} + +impl BoundaryConfig { + /// Serialize the protected driver-owned boundary configuration. + pub fn encode(&self) -> Result, BackendError> { + serde_json::to_vec(self) + .map_err(|error| BackendError::Descriptor(format!("encode boundary config: {error}"))) + } +} + +/// Validate driver-specific immutable coordinates before a boundary binds them. +/// +/// Claim values are opaque to the common protocol, but empty or +/// whitespace-bearing identifiers cannot safely distinguish runtime objects. +pub fn validate_resource_claims( + claims: &std::collections::BTreeMap, +) -> Result<(), BackendError> { + for (key, value) in claims { + if key.is_empty() || key.chars().any(char::is_whitespace) { + return Err(BackendError::Descriptor( + "boundary resource-claim keys must be non-empty and contain no whitespace" + .to_string(), + )); + } + if value.is_empty() || value.chars().any(char::is_whitespace) { + return Err(BackendError::Descriptor(format!( + "boundary resource claim {key:?} must be non-empty and contain no whitespace" + ))); + } + } + Ok(()) +} + +pub async fn write_stream_frame( + writer: &mut (impl AsyncWrite + Unpin), + channel: u8, + payload: &[u8], +) -> io::Result<()> { + if payload.len() > MAX_STREAM_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "boundary stream frame exceeds limit", + )); + } + writer.write_u8(channel).await?; + writer + .write_u32(payload.len().try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "boundary stream frame length overflow", + ) + })?) + .await?; + writer.write_all(payload).await?; + writer.flush().await +} + +pub async fn read_stream_frame( + reader: &mut (impl AsyncRead + Unpin), +) -> io::Result)>> { + let channel = match reader.read_u8().await { + Ok(channel) => channel, + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + Err(error) => return Err(error), + }; + let declared = reader.read_u32().await? as usize; + if declared > MAX_STREAM_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("boundary stream frame is too large: {declared} bytes"), + )); + } + let mut payload = vec![0; declared]; + reader.read_exact(&mut payload).await?; + Ok(Some((channel, payload))) +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RequestEnvelope { + pub request_id: u64, + pub boundary_id: String, + pub bootstrap_token: String, + pub request: Request, +} + +impl fmt::Debug for RequestEnvelope { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RequestEnvelope") + .field("request_id", &self.request_id) + .field("boundary_id", &self.boundary_id) + .field("bootstrap_token", &"") + .field("request", &self.request) + .finish() + } +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "operation", rename_all = "snake_case")] +pub enum Request { + Attach { + policy: Box, + resource_claims: std::collections::BTreeMap, + }, + Confirm, + StartAgent { + sandbox_id: String, + spec: AgentSpecWire, + policy: Box, + ca_cert: Option>, + ca_bundle: Option>, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + }, + UpdateProviderEnvironment { + expected_revision: u64, + revision: u64, + provider_env: std::collections::HashMap, + }, + AttachProcess { + process_id: String, + }, + Wait { + process_id: String, + }, + Signal { + process_id: String, + signal: SignalWire, + }, + Terminate { + process_id: String, + }, + Exec { + spec: ExecSpecWire, + }, + ExecSignal { + process_id: String, + signal: SignalWire, + }, + Resize { + process_id: String, + cols: u16, + rows: u16, + }, + PortForward { + host: std::net::IpAddr, + port: u16, + }, + AcceptNetwork, +} + +impl fmt::Debug for Request { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Attach { + policy: _, + resource_claims, + } => formatter + .debug_struct("Attach") + .field("policy", &"") + .field("resource_claims", resource_claims) + .finish(), + Self::Confirm => formatter.write_str("Confirm"), + Self::StartAgent { + sandbox_id, + spec, + policy: _, + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + } => formatter + .debug_struct("StartAgent") + .field("sandbox_id", sandbox_id) + .field("spec", spec) + .field("policy", &"") + .field("ca_cert_present", &ca_cert.is_some()) + .field("ca_bundle_present", &ca_bundle.is_some()) + .field("provider_env_revision", provider_env_revision) + .field( + "provider_env_keys", + &provider_env.keys().collect::>(), + ) + .finish(), + Self::UpdateProviderEnvironment { + expected_revision, + revision, + provider_env, + } => formatter + .debug_struct("UpdateProviderEnvironment") + .field("expected_revision", expected_revision) + .field("revision", revision) + .field( + "provider_env_keys", + &provider_env.keys().collect::>(), + ) + .finish(), + Self::Wait { process_id } => formatter + .debug_struct("Wait") + .field("process_id", process_id) + .finish(), + Self::AttachProcess { process_id } => formatter + .debug_struct("AttachProcess") + .field("process_id", process_id) + .finish(), + Self::Signal { process_id, signal } => formatter + .debug_struct("Signal") + .field("process_id", process_id) + .field("signal", signal) + .finish(), + Self::Terminate { process_id } => formatter + .debug_struct("Terminate") + .field("process_id", process_id) + .finish(), + Self::Exec { spec } => formatter.debug_tuple("Exec").field(spec).finish(), + Self::ExecSignal { process_id, signal } => formatter + .debug_struct("ExecSignal") + .field("process_id", process_id) + .field("signal", signal) + .finish(), + Self::Resize { + process_id, + cols, + rows, + } => formatter + .debug_struct("Resize") + .field("process_id", process_id) + .field("cols", cols) + .field("rows", rows) + .finish(), + Self::PortForward { host, port } => formatter + .debug_struct("PortForward") + .field("host", host) + .field("port", port) + .finish(), + Self::AcceptNetwork => formatter.write_str("AcceptNetwork"), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResponseEnvelope { + pub request_id: u64, + pub response: Response, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "snake_case")] +pub enum Response { + Attached, + Confirmed, + Started { + process_id: String, + provider_env_revision: u64, + }, + ProviderEnvironmentUpdated { + revision: u64, + }, + ProcessAttached { + terminal: bool, + }, + Exited { + status: ExitStatusWire, + }, + Signaled, + Terminated, + ExecStarted { + process_id: String, + pty: bool, + }, + Resized, + PortConnected, + NetworkConnected { + identity: BinaryIdentityWire, + }, + Error { + kind: String, + message: String, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BinaryIdentityWire { + pub binary_path: Option, + pub binary_digest: Option, + pub ancestors: Vec, + pub cmdline_paths: Vec, + pub resolve_error: Option, +} + +impl From> for BinaryIdentityWire { + fn from(identity: Result) -> Self { + match identity { + Ok(identity) => Self { + binary_path: Some(identity.binary_path), + binary_digest: identity.binary_digest.map(|digest| digest.to_string()), + ancestors: identity.ancestors, + cmdline_paths: identity.cmdline_paths, + resolve_error: None, + }, + Err(error) => Self { + binary_path: None, + binary_digest: None, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + resolve_error: Some(error.to_string()), + }, + } + } +} + +impl BinaryIdentityWire { + pub fn into_result(self) -> Result { + if let Some(error) = self.resolve_error { + return Err(ResolveError::Failed(error)); + } + let binary_path = self.binary_path.ok_or_else(|| { + ResolveError::Failed("boundary identity omitted binary path".to_string()) + })?; + let binary_digest = self + .binary_digest + .map(|digest| digest.parse::()) + .transpose()?; + Ok(BinaryIdentity { + binary_path, + binary_digest, + ancestors: self.ancestors, + cmdline_paths: self.cmdline_paths, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExecSpecWire { + pub program: String, + pub args: Vec, + pub env: Vec<(String, String)>, + pub workdir: Option, + pub pty: bool, +} + +impl From for ExecSpecWire { + fn from(spec: ExecSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + env: spec.env, + workdir: spec.workdir, + pty: spec.pty, + } + } +} + +impl From for ExecSpec { + fn from(spec: ExecSpecWire) -> Self { + Self { + program: spec.program, + args: spec.args, + env: spec.env, + workdir: spec.workdir, + pty: spec.pty, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentSpecWire { + pub program: String, + pub args: Vec, + pub workdir: Option, + pub timeout_secs: u64, + pub interactive: bool, +} + +impl From for AgentSpecWire { + fn from(spec: AgentSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + workdir: spec.workdir, + timeout_secs: spec.timeout_secs, + interactive: spec.interactive, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxPolicyWire { + pub version: u32, + pub read_only: Vec, + pub read_write: Vec, + pub include_workdir: bool, + pub network: NetworkModeWire, + pub proxy_addr: Option, + pub landlock: LandlockCompatibilityWire, + pub run_as_user: Option, + pub run_as_group: Option, +} + +impl From for SandboxPolicyWire { + fn from(policy: SandboxPolicy) -> Self { + // Exhaustively destructure the policy so adding a `SandboxPolicy` + // field is a compile error here instead of a silently dropped field + // across the host-to-guest trust boundary. + let SandboxPolicy { + version, + filesystem, + network, + landlock, + process, + } = policy; + let FilesystemPolicy { + read_only, + read_write, + include_workdir, + } = filesystem; + let NetworkPolicy { mode, proxy } = network; + let LandlockPolicy { compatibility } = landlock; + let ProcessPolicy { + run_as_user, + run_as_group, + } = process; + Self { + version, + read_only, + read_write, + include_workdir, + network: NetworkModeWire::from(mode), + proxy_addr: proxy.and_then(|proxy| proxy.http_addr), + landlock: LandlockCompatibilityWire::from(compatibility), + run_as_user, + run_as_group, + } + } +} + +impl From for SandboxPolicy { + fn from(policy: SandboxPolicyWire) -> Self { + let proxy = matches!(policy.network, NetworkModeWire::Proxy).then_some(ProxyPolicy { + http_addr: policy.proxy_addr, + }); + Self { + version: policy.version, + filesystem: FilesystemPolicy { + read_only: policy.read_only, + read_write: policy.read_write, + include_workdir: policy.include_workdir, + }, + network: NetworkPolicy { + mode: policy.network.into(), + proxy, + }, + landlock: LandlockPolicy { + compatibility: policy.landlock.into(), + }, + process: ProcessPolicy { + run_as_user: policy.run_as_user, + run_as_group: policy.run_as_group, + }, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NetworkModeWire { + Block, + Proxy, + Allow, +} + +impl From for NetworkModeWire { + fn from(mode: NetworkMode) -> Self { + match mode { + NetworkMode::Block => Self::Block, + NetworkMode::Proxy => Self::Proxy, + NetworkMode::Allow => Self::Allow, + } + } +} + +impl From for NetworkMode { + fn from(mode: NetworkModeWire) -> Self { + match mode { + NetworkModeWire::Block => Self::Block, + NetworkModeWire::Proxy => Self::Proxy, + NetworkModeWire::Allow => Self::Allow, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LandlockCompatibilityWire { + BestEffort, + HardRequirement, +} + +impl From for LandlockCompatibilityWire { + fn from(compatibility: LandlockCompatibility) -> Self { + match compatibility { + LandlockCompatibility::BestEffort => Self::BestEffort, + LandlockCompatibility::HardRequirement => Self::HardRequirement, + } + } +} + +impl From for LandlockCompatibility { + fn from(compatibility: LandlockCompatibilityWire) -> Self { + match compatibility { + LandlockCompatibilityWire::BestEffort => Self::BestEffort, + LandlockCompatibilityWire::HardRequirement => Self::HardRequirement, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignalWire { + Term, + Kill, + Int, + Hup, +} + +impl From for SignalWire { + fn from(signal: BoundarySignal) -> Self { + match signal { + BoundarySignal::Term => Self::Term, + BoundarySignal::Kill => Self::Kill, + BoundarySignal::Int => Self::Int, + BoundarySignal::Hup => Self::Hup, + } + } +} + +impl From for BoundarySignal { + fn from(signal: SignalWire) -> Self { + match signal { + SignalWire::Term => Self::Term, + SignalWire::Kill => Self::Kill, + SignalWire::Int => Self::Int, + SignalWire::Hup => Self::Hup, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum ExitStatusWire { + Exited(i32), + Signaled(i32), +} + +impl From for BoundaryExitStatus { + fn from(status: ExitStatusWire) -> Self { + match status { + ExitStatusWire::Exited(code) => Self::Exited(code), + ExitStatusWire::Signaled(signal) => Self::Signaled(signal), + } + } +} + +impl From for ExitStatusWire { + fn from(status: BoundaryExitStatus) -> Self { + match status { + BoundaryExitStatus::Exited(code) => Self::Exited(code), + BoundaryExitStatus::Signaled(signal) => Self::Signaled(signal), + } + } +} + +pub fn encode_frame(message: &T) -> Result, FrameError> { + let payload = serde_json::to_vec(message).map_err(FrameError::Serialize)?; + if payload.len() > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(payload.len())); + } + let length = u32::try_from(payload.len()).map_err(|_| FrameError::TooLarge(payload.len()))?; + let mut frame = Vec::with_capacity(4 + payload.len()); + frame.extend_from_slice(&length.to_be_bytes()); + frame.extend_from_slice(&payload); + Ok(frame) +} + +pub fn decode_frame(frame: &[u8]) -> Result { + let header: [u8; 4] = frame + .get(..4) + .ok_or(FrameError::Truncated)? + .try_into() + .map_err(|_| FrameError::Truncated)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let payload = frame.get(4..).ok_or(FrameError::Truncated)?; + if payload.len() != declared { + return Err(FrameError::LengthMismatch { + declared, + actual: payload.len(), + }); + } + serde_json::from_slice(payload).map_err(FrameError::Deserialize) +} + +pub fn read_frame(reader: &mut impl Read) -> Result { + let mut header = [0_u8; 4]; + reader.read_exact(&mut header)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + reader.read_exact(&mut frame[4..])?; + decode_frame(&frame) +} + +pub fn write_frame(writer: &mut impl Write, message: &T) -> Result<(), FrameError> { + let frame = encode_frame(message)?; + writer.write_all(&frame)?; + writer.flush()?; + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum FrameError { + #[error("control frame is truncated")] + Truncated, + #[error("control frame is too large: {0} bytes")] + TooLarge(usize), + #[error("control frame declared {declared} bytes but contained {actual}")] + LengthMismatch { declared: usize, actual: usize }, + #[error("serialize control frame: {0}")] + Serialize(serde_json::Error), + #[error("deserialize control frame: {0}")] + Deserialize(serde_json::Error), + #[error("read or write control frame: {0}")] + Io(#[from] io::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_round_trips_and_redacts_token() { + let request = RequestEnvelope { + request_id: 7, + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "never-log-this".to_string(), + request: Request::StartAgent { + sandbox_id: "sandbox-1".to_string(), + spec: AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + policy: Box::new(SandboxPolicyWire::from(SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + })), + ca_cert: Some(b"test certificate".to_vec()), + ca_bundle: Some(b"test bundle".to_vec()), + provider_env_revision: 7, + provider_env: std::collections::HashMap::from([( + "OPENAI_API_KEY".to_string(), + "test credential".to_string(), + )]), + }, + }; + let frame = encode_frame(&request).expect("encode request"); + let decoded: RequestEnvelope = decode_frame(&frame).expect("decode request"); + assert_eq!(decoded, request); + let debug = format!("{request:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + assert!(!debug.contains("test credential")); + assert!(!debug.contains("test certificate")); + assert!(!debug.contains("test bundle")); + assert!(debug.contains("OPENAI_API_KEY")); + } + + #[test] + fn rejects_declared_oversize() { + let oversized = u32::try_from(MAX_CONTROL_FRAME_BYTES + 1).expect("test size fits u32"); + let mut frame = Vec::from(oversized.to_be_bytes()); + frame.extend_from_slice(b"{}"); + assert!(matches!( + decode_frame::(&frame), + Err(FrameError::TooLarge(_)) + )); + } + + #[test] + fn resource_claims_reject_empty_or_ambiguous_identities() { + assert!( + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + String::new() + ),])) + .is_err() + ); + assert!( + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod uid".to_string(), + "uid-1".to_string() + ),])) + .is_err() + ); + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + "uid-1".to_string(), + )])) + .expect("opaque resource identity should be valid"); + } +} diff --git a/crates/openshell-isolation-interface/src/contract.rs b/crates/openshell-isolation-interface/src/contract.rs index 89fcf7ce48..52c0e60326 100644 --- a/crates/openshell-isolation-interface/src/contract.rs +++ b/crates/openshell-isolation-interface/src/contract.rs @@ -427,6 +427,13 @@ pub enum BoundarySignal { /// however many times it is called; a local PID is never the process handle. #[async_trait] pub trait BoundaryProcess: Send + Sync { + /// Attach to the admitted process's retained standard I/O. The boundary + /// remains the process owner and may permit only one control attachment. + async fn attach(&self) -> Result { + Err(BackendError::Unsupported( + "process attachment is not supported".to_string(), + )) + } /// Await terminal status (stable across repeated calls). async fn wait(&self) -> Result; /// Deliver a signal to the process or its group. @@ -440,6 +447,18 @@ pub type BoundaryInput = Box; /// A boxed async reader from a boundary process's stdout or stderr. pub type BoundaryOutput = Box; +/// A control-side attachment to the admitted process's retained I/O. +pub struct ProcessAttachment { + /// Stdin writer. + pub stdin: BoundaryInput, + /// Stdout reader, or the PTY-merged output stream. + pub stdout: BoundaryOutput, + /// Stderr reader, distinct from stdout for non-PTY processes. + pub stderr: Option, + /// PTY control, present when the admitted process owns a terminal. + pub terminal: Option>, +} + /// A PTY attached to an exec session. #[async_trait] pub trait BoundaryTerminal: Send + Sync { diff --git a/crates/openshell-isolation-interface/src/lib.rs b/crates/openshell-isolation-interface/src/lib.rs index d7f4e32183..8ec9d96e58 100644 --- a/crates/openshell-isolation-interface/src/lib.rs +++ b/crates/openshell-isolation-interface/src/lib.rs @@ -47,4 +47,8 @@ pub struct AgentSpec { pub interactive: bool, } +/// Versioned control-to-boundary wire types shared by every backend. +pub mod boundary_protocol; pub mod contract; +/// Reusable control-side implementation for a remote boundary endpoint. +pub mod remote; diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs new file mode 100644 index 0000000000..1055f6bcb9 --- /dev/null +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -0,0 +1,1321 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Host-side RFC 0012 backend for an already-provisioned remote boundary. + +#![allow(unsafe_code)] + +#[cfg(target_os = "linux")] +use std::mem::size_of; +#[cfg(target_os = "linux")] +use std::os::fd::{FromRawFd as _, IntoRawFd as _}; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use crate::AgentSpec; +use crate::contract::{ + BackendError, BoundBoundary, BoundaryDuplexStream, BoundaryExec, BoundaryExitStatus, + BoundaryInput, BoundaryOutput, BoundaryPortForward, BoundaryProcess, BoundarySignal, + BoundaryTerminal, ExecSession, ExecSpec, INTERFACE_VERSION, IsolationBackend, LoopbackTarget, + MediatedConnection, NetworkMediationSource, ProcessAttachment, ReadyBoundary, RunningBoundary, + SandboxContext, VerifiedTopologyDescriptor, +}; +use async_trait::async_trait; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::UnixStream; +use tokio::sync::Notify; + +use crate::boundary_protocol::{ + AgentSpecWire, BOUNDARY_PROTOCOL_VERSION, BoundaryTopology, BoundaryTransport, ExecSpecWire, + ExitStatusWire, MAX_CONTROL_FRAME_BYTES, Request, RequestEnvelope, Response, ResponseEnvelope, + STREAM_EXIT, STREAM_STDERR, STREAM_STDIN, STREAM_STDIN_CLOSED, STREAM_STDOUT, + SandboxPolicyWire, SignalWire, decode_frame, encode_frame, read_stream_frame, + validate_resource_claims, write_stream_frame, +}; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// How long one control call keeps retrying boundary connect attempts. Boot-time +/// callers retry whole calls above this; past boot, exhausting this window +/// means the remote boundary (or its launcher) is gone rather than still starting. +const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +const MIN_BOOTSTRAP_TOKEN_BYTES: usize = 32; + +/// Host-side remote boundary implementation registered with the supervisor. +#[derive(Debug)] +pub struct RemoteIsolationBackend { + backend_name: String, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, +} + +impl RemoteIsolationBackend { + pub fn new( + backend_name: impl Into, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, + ) -> Self { + Self { + backend_name: backend_name.into(), + ca_file_paths, + provider_credentials, + } + } +} + +#[async_trait] +impl IsolationBackend for RemoteIsolationBackend { + fn backend_name(&self) -> &str { + &self.backend_name + } + + fn version(&self) -> u32 { + INTERFACE_VERSION + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + let topology: BoundaryTopology = serde_json::from_slice(descriptor.payload()) + .map_err(|error| BackendError::Descriptor(format!("decode topology: {error}")))?; + validate_topology(&topology, &sandbox)?; + let host_gateway_ip = topology.host_gateway_ip; + let resource_claims = topology.resource_claims.clone(); + let client = Arc::new(BoundaryClient::new(topology)); + expect_response( + client + .call_idempotent(Request::Attach { + policy: Box::new(SandboxPolicyWire::from(sandbox.policy.clone())), + resource_claims, + }) + .await?, + "attached", + )?; + Ok(Box::new(RemoteBound { + client: client.clone(), + agent: sandbox.agent, + policy: sandbox.policy, + sandbox_id: sandbox.sandbox_id, + mediation: Arc::new(RemoteNetworkMediation { client }), + host_gateway_ip, + ca_file_paths: self.ca_file_paths.clone(), + provider_credentials: self.provider_credentials.clone(), + })) + } +} + +fn validate_topology( + topology: &BoundaryTopology, + sandbox: &SandboxContext, +) -> Result<(), BackendError> { + if topology.protocol_version != BOUNDARY_PROTOCOL_VERSION { + return Err(BackendError::Descriptor(format!( + "boundary protocol version {} unsupported (expected {BOUNDARY_PROTOCOL_VERSION})", + topology.protocol_version + ))); + } + if topology.boundary_id != sandbox.sandbox_id { + return Err(BackendError::Descriptor(format!( + "boundary {:?} does not match sandbox {:?}", + topology.boundary_id, sandbox.sandbox_id + ))); + } + if topology.bootstrap_token.len() < MIN_BOOTSTRAP_TOKEN_BYTES { + return Err(BackendError::Descriptor(format!( + "boundary bootstrap token must be at least {MIN_BOOTSTRAP_TOKEN_BYTES} bytes" + ))); + } + validate_resource_claims(&topology.resource_claims)?; + match &topology.transport { + BoundaryTransport::Unix { socket_path } => validate_socket_path(socket_path)?, + BoundaryTransport::Tcp { address } => { + if address.port() == 0 || address.ip().is_unspecified() { + return Err(BackendError::Descriptor( + "boundary TCP address must have a concrete IP and nonzero port".to_string(), + )); + } + } + BoundaryTransport::TlsTcp { + address, + server_name, + ca_certificate_pem, + } => { + validate_tcp_address(*address)?; + rustls::pki_types::ServerName::try_from(server_name.clone()).map_err(|error| { + BackendError::Descriptor(format!( + "boundary TLS server name {server_name:?} is invalid: {error}" + )) + })?; + tls_client_config(ca_certificate_pem)?; + } + BoundaryTransport::Vsock { + guest_cid, + control_port, + } => { + if *guest_cid < 3 { + return Err(BackendError::Descriptor( + "boundary CID must be at least 3".to_string(), + )); + } + validate_control_port(*control_port)?; + } + } + Ok(()) +} + +fn validate_tcp_address(address: std::net::SocketAddr) -> Result<(), BackendError> { + if address.port() == 0 || address.ip().is_unspecified() { + Err(BackendError::Descriptor( + "boundary TCP address must have a concrete IP and nonzero port".to_string(), + )) + } else { + Ok(()) + } +} + +fn tls_client_config(ca_certificate_pem: &str) -> Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let certificates = rustls_pemfile::certs(&mut ca_certificate_pem.as_bytes()) + .collect::, _>>() + .map_err(|error| { + BackendError::Descriptor(format!("parse boundary TLS CA certificate: {error}")) + })?; + if certificates.is_empty() { + return Err(BackendError::Descriptor( + "boundary TLS CA certificate PEM contains no certificates".to_string(), + )); + } + let mut roots = rustls::RootCertStore::empty(); + for certificate in certificates { + roots.add(certificate).map_err(|error| { + BackendError::Descriptor(format!("load boundary TLS CA certificate: {error}")) + })?; + } + Ok(rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth()) +} + +fn validate_socket_path(path: &std::path::Path) -> Result<(), BackendError> { + if path.is_absolute() { + Ok(()) + } else { + Err(BackendError::Descriptor( + "boundary control Unix socket path must be absolute".to_string(), + )) + } +} + +fn validate_control_port(port: u32) -> Result<(), BackendError> { + if port == 0 { + Err(BackendError::Descriptor( + "boundary control port must be nonzero".to_string(), + )) + } else { + Ok(()) + } +} + +struct RemoteBound { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + mediation: Arc, + host_gateway_ip: Option, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, +} + +#[async_trait] +impl BoundBoundary for RemoteBound { + fn network_mediation_source(&self) -> Arc { + self.mediation.clone() + } + + fn host_gateway_ip(&self) -> Option { + self.host_gateway_ip + } + + async fn confirm(self: Box) -> Result, BackendError> { + expect_response( + self.client.call_idempotent(Request::Confirm).await?, + "confirmed", + )?; + Ok(Box::new(RemoteReady { + client: self.client, + agent: self.agent, + policy: self.policy, + sandbox_id: self.sandbox_id, + ca_file_paths: self.ca_file_paths, + provider_credentials: self.provider_credentials, + })) + } +} + +struct RemoteReady { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, +} + +#[async_trait] +impl ReadyBoundary for RemoteReady { + async fn start_agent(self: Box) -> Result, BackendError> { + let ca_paths = self + .ca_file_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let (ca_cert, ca_bundle) = if let Some((ca_cert, ca_bundle)) = ca_paths { + let ca_cert = tokio::fs::read(&ca_cert).await.map_err(|error| { + BackendError::Process(format!("read host proxy CA {}: {error}", ca_cert.display())) + })?; + let ca_bundle = tokio::fs::read(&ca_bundle).await.map_err(|error| { + BackendError::Process(format!( + "read host proxy CA bundle {}: {error}", + ca_bundle.display() + )) + })?; + (Some(ca_cert), Some(ca_bundle)) + } else { + (None, None) + }; + let (provider_env_revision, provider_env) = self + .provider_credentials + .child_env_snapshot_with_gcp_resolved(); + let response = self + .client + .call(Request::StartAgent { + sandbox_id: self.sandbox_id, + spec: AgentSpecWire::from(self.agent), + policy: Box::new(SandboxPolicyWire::from(self.policy)), + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + }) + .await?; + let Response::Started { + process_id, + provider_env_revision, + } = response + else { + return Err(unexpected_response("started", &response)); + }; + let process = Arc::new(RemoteProcess { + client: self.client.clone(), + process_id, + }); + Ok(Box::new(RemoteRunning { + process, + exec: Arc::new(RemoteExec { + client: self.client.clone(), + provider_credentials: self.provider_credentials, + boundary_revision: tokio::sync::Mutex::new(provider_env_revision), + }), + port_forward: Arc::new(RemotePortForward { + client: self.client, + }), + })) + } +} + +struct RemoteRunning { + process: Arc, + exec: Arc, + port_forward: Arc, +} + +impl RunningBoundary for RemoteRunning { + fn agent(&self) -> Arc { + self.process.clone() + } + + fn exec(&self) -> Arc { + self.exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +struct RemoteProcess { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryProcess for RemoteProcess { + async fn attach(&self) -> Result { + open_process_attachment(self.client.clone(), self.process_id.clone()).await + } + + async fn wait(&self) -> Result { + let response = self + .client + .call_wait(Request::Wait { + process_id: self.process_id.clone(), + }) + .await + .map_err(|error| match error { + // A wait that can no longer reach the boundary leaf means the + // boundary is gone, not that a retry could still observe the + // exit status; report boundary loss per the contract. + BackendError::Unavailable(message) => { + BackendError::Terminated(format!("boundary lost during wait: {message}")) + } + error => error, + })?; + let Response::Exited { status } = response else { + return Err(unexpected_response("exited", &response)); + }; + Ok(status.into()) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Signal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?; + expect_response(response, "signaled") + } + + async fn terminate(&self) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Terminate { + process_id: self.process_id.clone(), + }) + .await?; + expect_response(response, "terminated") + } +} + +async fn open_process_attachment( + client: Arc, + process_id: String, +) -> Result { + let (stream, response) = client + .call_stream(Request::AttachProcess { + process_id: process_id.clone(), + }) + .await?; + let Response::ProcessAttached { + terminal: has_terminal, + } = response + else { + return Err(unexpected_response("process_attached", &response)); + }; + let (network_reader, network_writer) = tokio::io::split(stream); + let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); + let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); + let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); + tokio::spawn(pump_exec_input(stdin_pump, network_writer)); + tokio::spawn(pump_process_responses( + network_reader, + stdout_pump, + stderr_pump, + )); + let terminal: Option> = if has_terminal { + let terminal: Arc = Arc::new(RemoteTerminal { client, process_id }); + Some(terminal) + } else { + None + }; + let stderr: Option = if has_terminal { + None + } else { + let stderr: BoundaryOutput = Box::new(stderr); + Some(stderr) + }; + Ok(ProcessAttachment { + stdin: Box::new(stdin), + stdout: Box::new(stdout), + stderr, + terminal, + }) +} + +async fn pump_process_responses( + mut network: tokio::io::ReadHalf, + mut stdout: tokio::io::DuplexStream, + mut stderr: tokio::io::DuplexStream, +) { + loop { + match read_stream_frame(&mut network).await { + Ok(Some((STREAM_STDOUT, payload))) => { + if stdout.write_all(&payload).await.is_err() { + return; + } + } + Ok(Some((STREAM_STDERR, payload))) => { + if stderr.write_all(&payload).await.is_err() { + return; + } + } + Ok(Some((STREAM_EXIT, _)) | None) | Err(_) => return, + Ok(Some((_channel, _))) => return, + } + } +} + +struct RemoteExec { + client: Arc, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, + boundary_revision: tokio::sync::Mutex, +} + +#[async_trait] +impl BoundaryExec for RemoteExec { + async fn exec(&self, spec: ExecSpec) -> Result { + let mut boundary_revision = self.boundary_revision.lock().await; + for _ in 0..3 { + let (revision, provider_env) = self + .provider_credentials + .child_env_snapshot_with_gcp_resolved(); + let response = self + .client + .call_idempotent(Request::UpdateProviderEnvironment { + expected_revision: *boundary_revision, + revision, + provider_env, + }) + .await?; + let Response::ProviderEnvironmentUpdated { + revision: effective_revision, + } = response + else { + return Err(unexpected_response( + "provider_environment_updated", + &response, + )); + }; + *boundary_revision = effective_revision; + if effective_revision == revision { + return open_exec_session(self.client.clone(), spec).await; + } + } + Err(BackendError::Process( + "boundary provider environment changed concurrently during reconciliation".to_string(), + )) + } +} + +struct RemotePortForward { + client: Arc, +} + +#[async_trait] +impl BoundaryPortForward for RemotePortForward { + async fn connect(&self, target: LoopbackTarget) -> Result { + let (stream, response) = self + .client + .call_stream(Request::PortForward { + host: target.host(), + port: target.port(), + }) + .await?; + match response { + Response::PortConnected => Ok(stream), + response => Err(unexpected_response("port_connected", &response)), + } + } +} + +struct RemoteExecProcess { + client: Arc, + process_id: String, + exit: Arc, +} + +struct RemoteExit { + result: std::sync::Mutex>>, + changed: Notify, +} + +impl RemoteExit { + fn new() -> Self { + Self { + result: std::sync::Mutex::new(None), + changed: Notify::new(), + } + } + + fn set(&self, result: Result) { + let mut current = self + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current.is_none() { + *current = Some(result); + self.changed.notify_waiters(); + } + } + + async fn wait(&self) -> Result { + loop { + let changed = self.changed.notified(); + let result = self + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + if let Some(result) = result { + return result.map_err(BackendError::Terminated); + } + changed.await; + } + } +} + +#[async_trait] +impl BoundaryProcess for RemoteExecProcess { + async fn wait(&self) -> Result { + self.exit.wait().await + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + expect_response( + self.client + .call(Request::ExecSignal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?, + "signaled", + ) + } + + async fn terminate(&self) -> Result<(), BackendError> { + self.signal(BoundarySignal::Kill).await + } +} + +struct RemoteTerminal { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryTerminal for RemoteTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Resize { + process_id: self.process_id.clone(), + cols, + rows, + }) + .await?; + if matches!(response, Response::Resized) { + Ok(()) + } else { + Err(unexpected_response("resized", &response)) + } + } +} + +async fn open_exec_session( + client: Arc, + spec: ExecSpec, +) -> Result { + let (stream, response) = client + .call_stream(Request::Exec { + spec: ExecSpecWire::from(spec), + }) + .await?; + let Response::ExecStarted { process_id, pty } = response else { + return Err(unexpected_response("exec_started", &response)); + }; + let (network_reader, network_writer) = tokio::io::split(stream); + let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); + let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); + let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); + let exit = Arc::new(RemoteExit::new()); + tokio::spawn(pump_exec_input(stdin_pump, network_writer)); + tokio::spawn(pump_exec_responses( + network_reader, + stdout_pump, + stderr_pump, + exit.clone(), + )); + + let process: Arc = Arc::new(RemoteExecProcess { + client: client.clone(), + process_id: process_id.clone(), + exit, + }); + let terminal: Option> = if pty { + Some(Arc::new(RemoteTerminal { client, process_id })) + } else { + None + }; + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let stderr: Option = if pty { None } else { Some(Box::new(stderr)) }; + Ok(ExecSession { + process, + stdin: Some(stdin), + stdout, + stderr, + terminal, + }) +} + +async fn pump_exec_input( + mut input: tokio::io::DuplexStream, + mut network: tokio::io::WriteHalf, +) { + let mut buffer = vec![0; 16 * 1024]; + loop { + match input.read(&mut buffer).await { + Ok(0) => { + let _ = write_stream_frame(&mut network, STREAM_STDIN_CLOSED, &[]).await; + return; + } + Ok(read) => { + if write_stream_frame(&mut network, STREAM_STDIN, &buffer[..read]) + .await + .is_err() + { + return; + } + } + Err(_) => return, + } + } +} + +async fn pump_exec_responses( + mut network: tokio::io::ReadHalf, + mut stdout: tokio::io::DuplexStream, + mut stderr: tokio::io::DuplexStream, + exit: Arc, +) { + loop { + match read_stream_frame(&mut network).await { + Ok(Some((STREAM_STDOUT, payload))) => { + if stdout.write_all(&payload).await.is_err() { + exit.set(Err("boundary exec stdout consumer closed".to_string())); + return; + } + } + Ok(Some((STREAM_STDERR, payload))) => { + if stderr.write_all(&payload).await.is_err() { + exit.set(Err("boundary exec stderr consumer closed".to_string())); + return; + } + } + Ok(Some((STREAM_EXIT, payload))) => { + let result = serde_json::from_slice::(&payload) + .map(BoundaryExitStatus::from) + .map_err(|error| format!("decode boundary exec exit: {error}")); + exit.set(result); + return; + } + Ok(Some((channel, _))) => { + exit.set(Err(format!( + "boundary exec returned unexpected stream channel {channel}" + ))); + return; + } + Ok(None) => { + exit.set(Err( + "boundary exec stream closed before exit status".to_string() + )); + return; + } + Err(error) => { + exit.set(Err(format!("read boundary exec stream: {error}"))); + return; + } + } + } +} + +/// Pulls boundary proxy connections over one authenticated vsock stream each. +struct RemoteNetworkMediation { + client: Arc, +} + +#[async_trait] +impl NetworkMediationSource for RemoteNetworkMediation { + async fn accept(&self) -> Result { + let (stream, response) = self.client.open_exchange(Request::AcceptNetwork).await?; + let Response::NetworkConnected { identity } = response else { + return Err(unexpected_response("network_connected", &response)); + }; + Ok(MediatedConnection { + stream, + binary_identity: identity.into_result(), + destination: None, + }) + } +} + +struct BoundaryClient { + topology: BoundaryTopology, + next_request_id: AtomicU64, +} + +impl BoundaryClient { + fn new(topology: BoundaryTopology) -> Self { + Self { + topology, + next_request_id: AtomicU64::new(1), + } + } + + async fn call(&self, request: Request) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, self.exchange(request)) + .await + .map_err(|_| { + BackendError::Unavailable("boundary control request timed out".to_string()) + })? + } + + async fn call_idempotent(&self, request: Request) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, async { + loop { + match self.exchange(request.clone()).await { + Ok(response) => return Ok(response), + Err(BackendError::Unavailable(_)) => { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(error) => return Err(error), + } + } + }) + .await + .map_err(|_| { + BackendError::Unavailable( + "boundary idempotent control request timed out while waiting for remote boundary boot".to_string(), + ) + })? + } + + async fn call_wait(&self, request: Request) -> Result { + self.exchange(request).await + } + + async fn call_stream( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + tokio::time::timeout(REQUEST_TIMEOUT, self.open_exchange(request)) + .await + .map_err(|_| { + BackendError::Unavailable("boundary stream request timed out".to_string()) + })? + } + + async fn exchange(&self, request: Request) -> Result { + let (_, response) = self.open_exchange(request).await?; + Ok(response) + } + + async fn open_exchange( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + let request_id = self.next_request_id.fetch_add(1, Ordering::Relaxed); + let envelope = RequestEnvelope { + request_id, + boundary_id: self.topology.boundary_id.clone(), + bootstrap_token: self.topology.bootstrap_token.clone(), + request, + }; + let mut stream = self.connect_boundary().await?; + let frame = encode_frame(&envelope) + .map_err(|error| BackendError::Process(format!("encode control request: {error}")))?; + stream.write_all(&frame).await.map_err(|error| { + BackendError::Unavailable(format!("write boundary control request: {error}")) + })?; + let mut header = [0_u8; 4]; + stream.read_exact(&mut header).await.map_err(|error| { + BackendError::Unavailable(format!("read boundary control response header: {error}")) + })?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(BackendError::Process(format!( + "boundary control response is too large: {declared} bytes" + ))); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + stream.read_exact(&mut frame[4..]).await.map_err(|error| { + BackendError::Unavailable(format!("read boundary control response: {error}")) + })?; + let response: ResponseEnvelope = decode_frame(&frame) + .map_err(|error| BackendError::Process(format!("decode control response: {error}")))?; + if response.request_id != request_id { + return Err(BackendError::Process(format!( + "boundary response ID {} did not match request ID {request_id}", + response.request_id + ))); + } + let response = match response.response { + Response::Error { kind, message } => Err(guest_error(&kind, message)), + response => Ok(response), + }?; + Ok((stream, response)) + } + + async fn connect_boundary(&self) -> Result { + let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; + loop { + match self.connect_boundary_once().await { + Ok(stream) => return Ok(stream), + Err(error) if tokio::time::Instant::now() >= deadline => return Err(error), + Err(_) => tokio::time::sleep(Duration::from_millis(25)).await, + } + } + } + + async fn connect_boundary_once(&self) -> Result { + match &self.topology.transport { + BoundaryTransport::Unix { socket_path } => { + let stream = UnixStream::connect(socket_path).await.map_err(|error| { + BackendError::Unavailable(format!( + "connect to mapped boundary control socket {}: {error}", + socket_path.display() + )) + })?; + Ok(Box::new(stream)) + } + BoundaryTransport::Tcp { address } => { + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[*address]) + .await + .map_err(|error| { + BackendError::Unavailable(format!( + "connect to boundary TCP endpoint {address}: {error}" + )) + })?; + enable_boundary_tcp_keepalive(&stream); + Ok(Box::new(stream)) + } + BoundaryTransport::TlsTcp { + address, + server_name, + ca_certificate_pem, + } => { + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[*address]) + .await + .map_err(|error| { + BackendError::Unavailable(format!( + "connect to boundary TLS endpoint {address}: {error}" + )) + })?; + enable_boundary_tcp_keepalive(&stream); + let server_name = rustls::pki_types::ServerName::try_from(server_name.clone()) + .map_err(|error| { + BackendError::Descriptor(format!( + "boundary TLS server name {server_name:?} is invalid: {error}" + )) + })?; + let connector = tokio_rustls::TlsConnector::from(Arc::new(tls_client_config( + ca_certificate_pem, + )?)); + let stream = connector + .connect(server_name, stream) + .await + .map_err(|error| { + BackendError::Unavailable(format!( + "authenticate boundary TLS endpoint {address}: {error}" + )) + })?; + Ok(Box::new(stream)) + } + BoundaryTransport::Vsock { + guest_cid, + control_port, + } => connect_host_vsock(*guest_cid, *control_port), + } + } +} + +fn enable_boundary_tcp_keepalive(stream: &tokio::net::TcpStream) { + let keepalive = socket2::TcpKeepalive::new() + .with_time(Duration::from_secs(30)) + .with_interval(Duration::from_secs(10)); + let _ = socket2::SockRef::from(stream).set_tcp_keepalive(&keepalive); +} + +#[cfg(target_os = "linux")] +fn connect_host_vsock( + guest_cid: u32, + control_port: u32, +) -> Result { + let fd = unsafe { libc::socket(libc::AF_VSOCK, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) }; + if fd < 0 { + return Err(BackendError::Unavailable(format!( + "create host vsock: {}", + std::io::Error::last_os_error() + ))); + } + let fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(fd) }; + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|error| { + BackendError::Unavailable(format!("convert host vsock address family: {error}")) + })?; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: control_port, + svm_cid: guest_cid, + svm_zero: [0; 4], + }; + let address_length = + libc::socklen_t::try_from(size_of::()).map_err(|error| { + BackendError::Unavailable(format!("convert host vsock address length: {error}")) + })?; + let result = unsafe { + libc::connect( + std::os::fd::AsRawFd::as_raw_fd(&fd), + (&raw const address).cast::(), + address_length, + ) + }; + if result != 0 { + return Err(BackendError::Unavailable(format!( + "connect host vsock CID {guest_cid} port {control_port}: {}", + std::io::Error::last_os_error() + ))); + } + let stream = unsafe { std::os::unix::net::UnixStream::from_raw_fd(fd.into_raw_fd()) }; + stream.set_nonblocking(true).map_err(|error| { + BackendError::Unavailable(format!("set host vsock nonblocking: {error}")) + })?; + let stream = UnixStream::from_std(stream).map_err(|error| { + BackendError::Unavailable(format!("register host vsock with Tokio: {error}")) + })?; + Ok(Box::new(stream)) +} + +#[cfg(not(target_os = "linux"))] +fn connect_host_vsock( + _guest_cid: u32, + _control_port: u32, +) -> Result { + Err(BackendError::Unavailable( + "host AF_VSOCK transport is supported only on Linux".to_string(), + )) +} + +fn expect_response(response: Response, expected: &str) -> Result<(), BackendError> { + let matches = matches!( + (&response, expected), + (Response::Attached, "attached") + | (Response::Confirmed, "confirmed") + | (Response::Signaled, "signaled") + | (Response::Terminated, "terminated") + ); + if matches { + Ok(()) + } else { + Err(unexpected_response(expected, &response)) + } +} + +fn unexpected_response(expected: &str, response: &Response) -> BackendError { + BackendError::Process(format!( + "expected boundary response {expected:?}, received {response:?}" + )) +} + +fn guest_error(kind: &str, message: String) -> BackendError { + let message = format!("boundary process leaf: {message}"); + match kind { + "invalid" => BackendError::Descriptor(message), + "denied" => BackendError::Denied(message), + "unavailable" => BackendError::Unavailable(message), + "terminated" => BackendError::Terminated(message), + _ => BackendError::Process(message), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + use rcgen::generate_simple_self_signed; + + use super::*; + + #[tokio::test] + async fn boundary_tcp_connections_enable_keepalive() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let connected = tokio::spawn(async move { tokio::net::TcpStream::connect(address).await }); + let (_server, _) = listener.accept().await.unwrap(); + let client = connected.await.unwrap().unwrap(); + + enable_boundary_tcp_keepalive(&client); + + assert!(socket2::SockRef::from(&client).keepalive().unwrap()); + } + + struct TestCertificate { + ca_pem: String, + server_config: Arc, + } + + fn test_certificate(name: &str) -> TestCertificate { + let _ = rustls::crypto::ring::default_provider().install_default(); + let certified = + generate_simple_self_signed(vec![name.to_string()]).expect("generate test certificate"); + let certificate = certified.cert.der().clone(); + let private_key = + rustls::pki_types::PrivatePkcs8KeyDer::from(certified.key_pair.serialize_der()); + let server_config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![certificate], private_key.into()) + .expect("build test TLS server config"); + TestCertificate { + ca_pem: certified.cert.pem(), + server_config: Arc::new(server_config), + } + } + + fn tls_topology( + address: std::net::SocketAddr, + server_name: &str, + ca_certificate_pem: String, + token: &str, + ) -> BoundaryTopology { + BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + transport: BoundaryTransport::TlsTcp { + address, + server_name: server_name.to_string(), + ca_certificate_pem, + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + bootstrap_token: token.to_string(), + } + } + + async fn spawn_tls_boundary( + certificate: Arc, + expected_token: String, + ) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test TLS boundary"); + let address = listener.local_addr().expect("read test listener address"); + let task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept TLS control client"); + let Ok(mut stream) = tokio_rustls::TlsAcceptor::from(certificate) + .accept(stream) + .await + else { + return; + }; + let declared_u32 = stream.read_u32().await.expect("read request length"); + let declared = declared_u32 as usize; + let mut frame = vec![0_u8; 4 + declared]; + frame[..4].copy_from_slice(&declared_u32.to_be_bytes()); + stream + .read_exact(&mut frame[4..]) + .await + .expect("read request frame"); + let request: RequestEnvelope = decode_frame(&frame).expect("decode request"); + let response = if request.boundary_id == "sandbox-1" + && request.bootstrap_token == expected_token + { + Response::Confirmed + } else { + Response::Error { + kind: "denied".to_string(), + message: "control authentication failed".to_string(), + } + }; + let frame = encode_frame(&ResponseEnvelope { + request_id: request.request_id, + response, + }) + .expect("encode response"); + stream.write_all(&frame).await.expect("write response"); + }); + (address, task) + } + + fn sandbox() -> SandboxContext { + SandboxContext { + sandbox_id: "sandbox-1".to_string(), + policy: SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }, + agent: AgentSpec { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + } + } + + #[test] + fn topology_debug_redacts_token() { + let topology = BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from("/tmp/vsock.sock"), + }, + host_gateway_ip: Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + resource_claims: std::collections::BTreeMap::new(), + bootstrap_token: "never-log-this-never-log-this".to_string(), + }; + let debug = format!("{topology:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn topology_must_match_sandbox() { + let topology = BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "other".to_string(), + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from("/tmp/vsock.sock"), + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + assert!(matches!( + validate_topology(&topology, &sandbox()), + Err(BackendError::Descriptor(_)) + )); + } + + #[test] + fn topology_rejects_an_unspecified_tcp_target() { + let topology = BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + transport: BoundaryTransport::Tcp { + address: "0.0.0.0:5500".parse().expect("valid address"), + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + assert!(matches!( + validate_topology(&topology, &sandbox()), + Err(BackendError::Descriptor(_)) + )); + } + + #[test] + fn topology_accepts_a_concrete_tcp_target() { + let topology = BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + transport: BoundaryTransport::Tcp { + address: "10.42.0.7:5500".parse().expect("valid address"), + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + validate_topology(&topology, &sandbox()).expect("TCP topology should be valid"); + } + + #[test] + fn topology_rejects_invalid_tls_configuration() { + let topology = tls_topology( + "127.0.0.1:5500".parse().expect("valid address"), + "not a dns name!", + "not a certificate".to_string(), + "0123456789abcdef0123456789abcdef", + ); + assert!(matches!( + validate_topology(&topology, &sandbox()), + Err(BackendError::Descriptor(_)) + )); + } + + #[tokio::test] + async fn tls_tcp_round_trip_verifies_server_certificate() { + let certificate = test_certificate("boundary.test"); + let (address, server) = spawn_tls_boundary(certificate.server_config, "a".repeat(32)).await; + let client = BoundaryClient::new(tls_topology( + address, + "boundary.test", + certificate.ca_pem, + &"a".repeat(32), + )); + + assert_eq!( + client.call(Request::Confirm).await.expect("TLS request"), + Response::Confirmed + ); + server.await.expect("TLS test server"); + } + + #[tokio::test] + async fn tls_tcp_preserves_boundary_token_authentication() { + let certificate = test_certificate("boundary.test"); + let (address, server) = spawn_tls_boundary( + certificate.server_config, + "expected-token-expected-token-12".to_string(), + ) + .await; + let client = BoundaryClient::new(tls_topology( + address, + "boundary.test", + certificate.ca_pem, + "incorrect-token-incorrect-token", + )); + + assert!(matches!( + client.call(Request::Confirm).await, + Err(BackendError::Denied(_)) + )); + server.await.expect("TLS test server"); + } + + #[tokio::test] + async fn tls_tcp_rejects_an_untrusted_server_certificate() { + let presented = test_certificate("boundary.test"); + let trusted = test_certificate("boundary.test"); + let (address, server) = spawn_tls_boundary(presented.server_config, "a".repeat(32)).await; + let client = BoundaryClient::new(tls_topology( + address, + "boundary.test", + trusted.ca_pem, + &"a".repeat(32), + )); + + assert!(matches!( + client.connect_boundary_once().await, + Err(BackendError::Unavailable(_)) + )); + // The server observes the client's fatal alert and may fail its accept; + // completing the task is sufficient for this rejection test. + let _ = server.await; + } +} diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 3463f03767..4b7cd5e99c 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-extension-core = { path = "../openshell-extension-core" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } @@ -42,8 +43,11 @@ nix = { workspace = true } # TLS crypto provider install (main.rs) rustls = { workspace = true } +rustls-pemfile = { workspace = true } +tokio-rustls = { workspace = true } # Serialization (serde_json::json! for OCSF unmapped fields) +base64 = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } prost = { workspace = true } @@ -54,6 +58,9 @@ uuid = { workspace = true } tracing-subscriber = { workspace = true } tracing-appender = { workspace = true } +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2" + [features] default = ["telemetry", "bundled-ca-roots"] ## Convenience alias: all defaults except bundled CA roots. Use @@ -76,6 +83,7 @@ telemetry = ["openshell-core/telemetry"] bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] [dev-dependencies] +rcgen = { workspace = true } tempfile = "3" temp-env = "0.3" tokio-tungstenite = { workspace = true } diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs new file mode 100644 index 0000000000..98882d2624 --- /dev/null +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -0,0 +1,2579 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared implementation of `openshell-sandbox --mode=boundary`. +//! +//! This is transport and lifecycle glue, not another supervisor model. When +//! the control role authorizes `start_agent`, it invokes the existing process +//! supervisor inside the driver-provisioned boundary. + +#![allow(unsafe_code)] + +use std::path::Path; + +#[cfg(target_os = "linux")] +mod linux { + use super::Path; + use std::ffi::CString; + use std::fs::File; + use std::io::{self, Read, Write}; + use std::mem::size_of; + use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd as _, OwnedFd}; + use std::os::unix::ffi::OsStrExt as _; + use std::os::unix::fs::PermissionsExt as _; + use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; + use std::sync::{Arc, Condvar, Mutex}; + use std::time::Duration; + + use openshell_core::proposals::AgentProposals; + use openshell_core::provider_credentials::ProviderCredentialState; + use openshell_isolation_interface::contract::{ + BoundaryExec, BoundaryPortForward, BoundaryProcess, BoundaryTerminal, ExecSession, + LoopbackTarget, + }; + use openshell_supervisor_network::identity_source::ProcfsIdentityResolver; + use openshell_supervisor_process::boundary_io::BoundaryRuntimeState; + use openshell_supervisor_process::delegated::{AgentSignaler, spawn_workload}; + use openshell_supervisor_process::identity::{DriverIdentity, resolve_process_identity}; + use openshell_supervisor_process::main_session::{MainOutput, MainSession}; + use openshell_supervisor_process::netns::{ + NetworkNamespace, create_conformant_netns_for_proxy, + }; + use openshell_supervisor_process::process::{ + ProcessEnforcementMode, ProcessStatus, ResolvedProcessIdentity, + }; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + use openshell_isolation_interface::boundary_protocol::{ + AgentSpecWire, BOUNDARY_PROTOCOL_VERSION, BinaryIdentityWire, BoundaryAgentIdentity, + BoundaryConfig, BoundaryListener as BoundaryListenerConfig, ExecSpecWire, ExitStatusWire, + Request, RequestEnvelope, Response, ResponseEnvelope, STREAM_EXIT, STREAM_STDERR, + STREAM_STDIN, STREAM_STDIN_CLOSED, STREAM_STDOUT, SandboxPolicyWire, SignalWire, + encode_frame, read_frame, read_stream_frame, validate_resource_claims, write_frame, + write_stream_frame, + }; + + const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30); + const OUTPUT_DRAIN_TIMEOUT: Duration = Duration::from_secs(2); + const MAX_CONTROL_CONNECTIONS: usize = 128; + + struct ControlConnectionSlot(Arc); + + impl Drop for ControlConnectionSlot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } + } + + fn acquire_control_connection_slot(active: &Arc) -> Option { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < MAX_CONTROL_CONNECTIONS).then_some(current + 1) + }) + .ok() + .map(|_| ControlConnectionSlot(active.clone())) + } + static BOUNDARY_TERMINATION_REQUESTED: AtomicBool = AtomicBool::new(false); + + extern "C" fn request_boundary_termination(_signal: libc::c_int) { + BOUNDARY_TERMINATION_REQUESTED.store(true, Ordering::Release); + } + + pub fn run_boundary(config_path: &Path) -> Result<(), String> { + install_boundary_signal_handlers()?; + if std::process::id() == 1 { + prepare_pid1_filesystems()?; + } + let bytes = std::fs::read(config_path) + .map_err(|error| format!("read boundary config {}: {error}", config_path.display()))?; + let config: BoundaryConfig = serde_json::from_slice(&bytes).map_err(|error| { + format!("decode boundary config {}: {error}", config_path.display()) + })?; + validate_config(&config)?; + if config.protect_config_file { + protect_boundary_config_file(config_path)?; + } + openshell_supervisor_process::netns::configure_trusted_runtime_root( + config.trusted_runtime_root.clone(), + ) + .map_err(|error| format!("configure trusted boundary helper runtime: {error}"))?; + let child_env = serde_json::to_string(&config.child_env) + .map_err(|error| format!("encode boundary workload environment: {error}"))?; + // This runs before the Tokio runtime or control threads exist. The process + // supervisor consumes the serialized map and applies values only to + // workload children. + unsafe { + std::env::set_var(openshell_core::sandbox_env::USER_ENVIRONMENT, child_env); + } + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| format!("create boundary process runtime: {error}"))?; + let runtime = Arc::new(BoundaryRuntime::new( + config.clone(), + process_runtime.handle().clone(), + )); + serve(&config.listener, runtime) + } + + fn install_boundary_signal_handlers() -> Result<(), String> { + BOUNDARY_TERMINATION_REQUESTED.store(false, Ordering::Release); + let action = nix::sys::signal::SigAction::new( + nix::sys::signal::SigHandler::Handler(request_boundary_termination), + nix::sys::signal::SaFlags::empty(), + nix::sys::signal::SigSet::empty(), + ); + for signal in [ + nix::sys::signal::Signal::SIGTERM, + nix::sys::signal::Signal::SIGINT, + ] { + // SAFETY: the installed handler only performs a lock-free atomic + // store, which is async-signal-safe, and remains valid for the + // lifetime of the boundary process. + unsafe { nix::sys::signal::sigaction(signal, &action) } + .map_err(|error| format!("install boundary {signal:?} handler: {error}"))?; + } + Ok(()) + } + + fn protect_boundary_config_file(config_path: &Path) -> Result<(), String> { + let path = CString::new(config_path.as_os_str().as_bytes()).map_err(|error| { + format!( + "boundary config path {} contains NUL: {error}", + config_path.display() + ) + })?; + if unsafe { libc::chown(path.as_ptr(), 0, 0) } != 0 { + return Err(format!( + "re-own boundary config {} as root: {}", + config_path.display(), + io::Error::last_os_error() + )); + } + std::fs::set_permissions(config_path, std::fs::Permissions::from_mode(0o600)).map_err( + |error| { + format!( + "restrict boundary config {} to root: {error}", + config_path.display() + ) + }, + ) + } + + fn validate_config(config: &BoundaryConfig) -> Result<(), String> { + if config.protocol_version != BOUNDARY_PROTOCOL_VERSION { + return Err(format!( + "boundary protocol version {} unsupported (expected {BOUNDARY_PROTOCOL_VERSION})", + config.protocol_version + )); + } + if config.boundary_id.is_empty() { + return Err("boundary ID must not be empty".to_string()); + } + if config.bootstrap_token.len() < 32 { + return Err("boundary bootstrap token must contain at least 32 bytes".to_string()); + } + validate_resource_claims(&config.resource_claims).map_err(|error| error.to_string())?; + match &config.listener { + BoundaryListenerConfig::Unix { socket_path } if !socket_path.is_absolute() => { + return Err("boundary Unix socket path must be absolute".to_string()); + } + BoundaryListenerConfig::Tcp { address } if address.port() == 0 => { + return Err("boundary TCP listener port must be nonzero".to_string()); + } + BoundaryListenerConfig::TlsTcp { + address, + certificate_chain_path, + private_key_path, + } if address.port() == 0 + || !certificate_chain_path.is_absolute() + || !private_key_path.is_absolute() => + { + return Err( + "boundary TLS listener requires a nonzero port and absolute certificate paths" + .to_string(), + ); + } + BoundaryListenerConfig::Vsock { control_port: 0 } => { + return Err("boundary control port must be nonzero".to_string()); + } + BoundaryListenerConfig::Unix { .. } + | BoundaryListenerConfig::Tcp { .. } + | BoundaryListenerConfig::TlsTcp { .. } + | BoundaryListenerConfig::Vsock { .. } => {} + } + match &config.agent_identity { + BoundaryAgentIdentity::Resolved { uid, gid } if *uid == 0 || *gid == 0 => { + return Err("boundary agent UID and GID must be nonzero".to_string()); + } + BoundaryAgentIdentity::Resolved { .. } + | BoundaryAgentIdentity::OciUser { .. } + | BoundaryAgentIdentity::None => {} + } + if !config.trusted_runtime_root.is_absolute() { + return Err("boundary trusted helper runtime root must be absolute".to_string()); + } + Ok(()) + } + + fn prepare_pid1_filesystems() -> Result<(), String> { + for path in ["/proc", "/sys", "/dev", "/run", "/tmp", "/sandbox"] { + std::fs::create_dir_all(path).map_err(|error| format!("create {path}: {error}"))?; + } + mount_if_needed("proc", "/proc", "proc")?; + mount_if_needed("sysfs", "/sys", "sysfs")?; + mount_if_needed("devtmpfs", "/dev", "devtmpfs")?; + std::fs::create_dir_all("/dev/pts").map_err(|error| format!("create /dev/pts: {error}"))?; + mount_if_needed("devpts", "/dev/pts", "devpts")?; + Ok(()) + } + + fn mount_if_needed(source: &str, target: &str, file_system: &str) -> Result<(), String> { + let source = CString::new(source).map_err(|error| error.to_string())?; + let target_c = CString::new(target).map_err(|error| error.to_string())?; + let file_system = CString::new(file_system).map_err(|error| error.to_string())?; + let result = unsafe { + libc::mount( + source.as_ptr(), + target_c.as_ptr(), + file_system.as_ptr(), + 0, + std::ptr::null(), + ) + }; + if result == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::EBUSY) { + Ok(()) + } else { + Err(format!("mount {file_system:?} on {target}: {error}")) + } + } + + fn serve(config: &BoundaryListenerConfig, runtime: Arc) -> Result<(), String> { + let listener = ControlListener::bind(config) + .map_err(|error| format!("bind boundary control listener: {error}"))?; + let active_connections = Arc::new(AtomicUsize::new(0)); + tracing::info!(?config, "Boundary control listener ready"); + loop { + if BOUNDARY_TERMINATION_REQUESTED.load(Ordering::Acquire) { + runtime.shutdown(); + return Ok(()); + } + match listener.accept() { + Ok(stream) => { + let Some(slot) = acquire_control_connection_slot(&active_connections) else { + tracing::warn!( + limit = MAX_CONTROL_CONNECTIONS, + "Boundary control connection limit reached" + ); + continue; + }; + let runtime = runtime.clone(); + std::thread::spawn(move || { + let _slot = slot; + let stream = match stream.establish(&runtime.process_runtime) { + Ok(stream) => stream, + Err(error) => { + tracing::warn!(%error, "Boundary control transport handshake failed"); + return; + } + }; + if let Err(error) = serve_one(stream, &runtime) { + tracing::warn!(%error, "Boundary control request failed"); + } + }); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(format!("accept boundary control connection: {error}")), + } + } + } + + fn serve_one(mut stream: ControlStream, runtime: &BoundaryRuntime) -> Result<(), String> { + stream + .set_timeout(CONTROL_IO_TIMEOUT) + .map_err(|error| format!("set control timeout: {error}"))?; + let request: RequestEnvelope = + read_frame(&mut stream).map_err(|error| format!("read control frame: {error}"))?; + if !runtime.authenticate(&request) { + let response = ResponseEnvelope { + request_id: request.request_id, + response: guest_error("denied", "control authentication failed"), + }; + return write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}")); + } + match request.request.clone() { + Request::Exec { spec } => { + let (process_id, session) = match runtime.start_exec(spec) { + Ok(started) => started, + Err(response) => { + return write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response, + }, + ) + .map_err(|error| format!("write exec error response: {error}")); + } + }; + if let Err(error) = write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::ExecStarted { + process_id: process_id.clone(), + pty: session.terminal.is_some(), + }, + }, + ) { + runtime.cancel_exec_start(&process_id, session.process.clone()); + return Err(format!("write exec start response: {error}")); + } + return runtime.stream_exec(stream, &process_id, session); + } + Request::AttachProcess { process_id } => { + let (attachment, terminal) = match runtime.attach_process(&process_id) { + Ok(attachment) => attachment, + Err(response) => { + return write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response, + }, + ) + .map_err(|error| format!("write process attachment error: {error}")); + } + }; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::ProcessAttached { terminal }, + }, + ) + .map_err(|error| format!("write process attachment response: {error}"))?; + return runtime.stream_process(stream, attachment); + } + Request::PortForward { host, port } => { + let target = LoopbackTarget::new(host, port) + .map_err(|error| format!("validate port-forward target: {error}"))?; + let mut target = runtime + .connect_port(target) + .map_err(|error| format!("connect boundary loopback port: {error}"))?; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::PortConnected, + }, + ) + .map_err(|error| format!("write port-forward response: {error}"))?; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + tokio::io::copy_bidirectional(&mut stream, &mut target) + .await + .map_err(|error| format!("bridge boundary loopback stream: {error}")) + })?; + return Ok(()); + } + Request::AcceptNetwork => { + let (listener, process) = runtime.network_accept_context()?; + let request_id = request.request_id; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + let (mut target, workload_addr) = + accept_network_while_control_connected(&listener, &mut stream).await?; + let proxy_addr = target + .local_addr() + .map_err(|error| format!("read boundary proxy address: {error}"))?; + let identity = BinaryIdentityWire::from( + process + .identity_resolver() + .resolve_connection(workload_addr, proxy_addr), + ); + let response = encode_frame(&ResponseEnvelope { + request_id, + response: Response::NetworkConnected { identity }, + }) + .map_err(|error| format!("encode network mediation response: {error}"))?; + stream + .write_all(&response) + .await + .map_err(|error| format!("write network mediation response: {error}"))?; + tokio::io::copy_bidirectional(&mut stream, &mut target) + .await + .map_err(|error| { + format!("bridge boundary network mediation stream: {error}") + }) + })?; + return Ok(()); + } + _ => {} + } + let response = ResponseEnvelope { + request_id: request.request_id, + response: runtime.dispatch(request), + }; + write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}"))?; + Ok(()) + } + + async fn accept_network_while_control_connected( + listener: &tokio::net::TcpListener, + control: &mut openshell_isolation_interface::contract::BoundaryDuplexStream, + ) -> Result<(tokio::net::TcpStream, std::net::SocketAddr), String> { + let mut unexpected_control_data = [0_u8; 1]; + tokio::select! { + biased; + control_result = control.read(&mut unexpected_control_data) => match control_result { + Ok(0) => Err("AcceptNetwork cancelled when control disconnected".to_string()), + Ok(_) => Err("unexpected data after AcceptNetwork request".to_string()), + Err(error) => Err(format!("monitor AcceptNetwork control stream: {error}")), + }, + accepted = listener.accept() => accepted.map_err(|error| { + format!("accept boundary proxy connection: {error}") + }), + } + } + + struct BoundaryRuntime { + config: BoundaryConfig, + process_runtime: tokio::runtime::Handle, + state: Mutex, + /// The wire policy bound at first attach, so an idempotent attach retry + /// carrying a different policy is denied instead of silently keeping + /// the first policy. + attached_policy: Mutex>, + /// The complete launch request accepted by the boundary. A replacement + /// control process may replay it after reconnecting, but may not change + /// any launch input or start a second workload. + started_agent: Mutex>, + next_exec_id: AtomicU64, + exec_handles: Mutex>, + } + + struct ExecHandle { + process: Arc, + terminal: Option>, + } + + #[derive(Clone, PartialEq, Eq)] + struct StartedAgent { + sandbox_id: String, + spec: AgentSpecWire, + policy: SandboxPolicyWire, + ca_cert: Option>, + ca_bundle: Option>, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + } + + impl StartedAgent { + /// Provider environment is mutable runtime state. A replacement + /// control must replay every immutable launch input exactly, then + /// reconcile the current provider snapshot through the CAS update. + fn matches_replay(&self, other: &Self) -> bool { + self.sandbox_id == other.sandbox_id + && self.spec == other.spec + && self.policy == other.policy + && self.ca_cert == other.ca_cert + && self.ca_bundle == other.ca_bundle + } + } + + struct MainAttachment { + process: Arc, + session: Arc, + } + + impl Drop for MainAttachment { + fn drop(&mut self) { + self.process.attached.store(false, Ordering::Release); + } + } + + enum RuntimeState { + AwaitingAttach, + Bound(PreparedBoundary), + Ready(PreparedBoundary), + Running(Arc), + } + + #[derive(Clone)] + struct PreparedBoundary { + netns: Option>, + network_listener: Option>, + proxy_port: u16, + } + + async fn bridge_exec_stream( + stream: openshell_isolation_interface::contract::BoundaryDuplexStream, + session: ExecSession, + ) -> Result<(), String> { + let ExecSession { + process, + stdin, + stdout, + stderr, + terminal: _, + } = session; + let (mut network_reader, network_writer) = tokio::io::split(stream); + let network_writer = Arc::new(tokio::sync::Mutex::new(network_writer)); + + let stdin_task = stdin.map(|mut stdin| { + tokio::spawn(async move { + while let Some((channel, payload)) = read_stream_frame(&mut network_reader).await? { + match channel { + STREAM_STDIN => stdin.write_all(&payload).await?, + STREAM_STDIN_CLOSED => { + stdin.shutdown().await?; + break; + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected host-to-boundary stream channel", + )); + } + } + } + Ok::<(), io::Error>(()) + }) + }); + + let stdout_task = tokio::spawn(pump_exec_output( + stdout, + STREAM_STDOUT, + network_writer.clone(), + )); + let stderr_task = stderr.map(|stderr| { + tokio::spawn(pump_exec_output( + stderr, + STREAM_STDERR, + network_writer.clone(), + )) + }); + + let status = process + .wait() + .await + .map_err(|error| format!("wait for boundary exec: {error}"))?; + drain_exec_output(stdout_task, "stdout").await?; + if let Some(stderr_task) = stderr_task { + drain_exec_output(stderr_task, "stderr").await?; + } + let exit = serde_json::to_vec(&ExitStatusWire::from(status)) + .map_err(|error| format!("encode boundary exec exit: {error}"))?; + write_stream_frame(&mut *network_writer.lock().await, STREAM_EXIT, &exit) + .await + .map_err(|error| format!("write boundary exec exit: {error}"))?; + if let Some(stdin_task) = stdin_task { + stdin_task.abort(); + } + Ok(()) + } + + async fn drain_exec_output( + mut task: tokio::task::JoinHandle>, + name: &str, + ) -> Result<(), String> { + if let Ok(result) = tokio::time::timeout(OUTPUT_DRAIN_TIMEOUT, &mut task).await { + result + .map_err(|error| format!("join boundary exec {name}: {error}"))? + .map_err(|error| format!("stream boundary exec {name}: {error}")) + } else { + task.abort(); + tracing::warn!(stream = name, "boundary exec output drain timed out"); + Ok(()) + } + } + + async fn pump_exec_output( + mut output: openshell_isolation_interface::contract::BoundaryOutput, + channel: u8, + writer: Arc< + tokio::sync::Mutex< + tokio::io::WriteHalf, + >, + >, + ) -> io::Result<()> { + let mut buffer = vec![0; 16 * 1024]; + loop { + let read = output.read(&mut buffer).await?; + if read == 0 { + return Ok(()); + } + write_stream_frame(&mut *writer.lock().await, channel, &buffer[..read]).await?; + } + } + + impl BoundaryRuntime { + fn new(config: BoundaryConfig, process_runtime: tokio::runtime::Handle) -> Self { + Self { + config, + process_runtime, + state: Mutex::new(RuntimeState::AwaitingAttach), + attached_policy: Mutex::new(None), + started_agent: Mutex::new(None), + next_exec_id: AtomicU64::new(1), + exec_handles: Mutex::new(std::collections::HashMap::new()), + } + } + + fn shutdown(&self) { + let process = { + let state = lock(&self.state); + match &*state { + RuntimeState::Running(process) => Some(process.clone()), + RuntimeState::AwaitingAttach + | RuntimeState::Bound(_) + | RuntimeState::Ready(_) => None, + } + }; + if let Some(process) = process { + process.boundary_runtime.deactivate(); + } + } + + fn dispatch(&self, envelope: RequestEnvelope) -> Response { + if !self.authenticate(&envelope) { + return guest_error("denied", "control authentication failed"); + } + match envelope.request { + Request::Attach { + policy, + resource_claims, + } => { + if resource_claims == self.config.resource_claims { + self.attach(*policy) + } else { + guest_error( + "denied", + "topology resource claims do not match the boundary configuration", + ) + } + } + Request::Confirm => self.confirm(), + Request::StartAgent { + sandbox_id, + spec, + policy, + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + } => self.start_agent( + sandbox_id, + spec, + *policy, + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + ), + Request::UpdateProviderEnvironment { + expected_revision, + revision, + provider_env, + } => self.update_provider_environment(expected_revision, revision, provider_env), + Request::Wait { process_id } => self.wait(&process_id), + Request::Signal { process_id, signal } => self.signal(&process_id, signal), + Request::Terminate { process_id } => self.terminate(&process_id), + Request::ExecSignal { process_id, signal } => self.signal_exec(&process_id, signal), + Request::Resize { + process_id, + cols, + rows, + } => self.resize_process(&process_id, cols, rows), + Request::Exec { .. } + | Request::AttachProcess { .. } + | Request::PortForward { .. } + | Request::AcceptNetwork => { + guest_error("invalid", "streaming request used on control path") + } + } + } + + fn authenticate(&self, envelope: &RequestEnvelope) -> bool { + constant_time_eq( + envelope.boundary_id.as_bytes(), + self.config.boundary_id.as_bytes(), + ) && constant_time_eq( + envelope.bootstrap_token.as_bytes(), + self.config.bootstrap_token.as_bytes(), + ) + } + + fn start_exec(&self, spec: ExecSpecWire) -> Result<(String, ExecSession), Response> { + let executor = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + process.boundary_exec() + }; + let session = self + .process_runtime + .block_on(executor.exec(spec.into())) + .map_err(|error| guest_error("failed", error.to_string()))?; + let process_id = format!("exec-{}", self.next_exec_id.fetch_add(1, Ordering::Relaxed)); + lock(&self.exec_handles).insert( + process_id.clone(), + ExecHandle { + process: session.process.clone(), + terminal: session.terminal.clone(), + }, + ); + Ok((process_id, session)) + } + + fn signal_exec(&self, process_id: &str, signal: SignalWire) -> Response { + let process = lock(&self.exec_handles) + .get(process_id) + .map(|handle| handle.process.clone()); + let Some(process) = process else { + return guest_error("invalid", "unknown exec process ID"); + }; + match self.process_runtime.block_on(process.signal(signal.into())) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("failed", error.to_string()), + } + } + + fn cancel_exec_start(&self, process_id: &str, process: Arc) { + lock(&self.exec_handles).remove(process_id); + self.process_runtime.block_on(async move { + let _ = process.terminate().await; + let _ = process.wait().await; + }); + } + + fn resize_process(&self, process_id: &str, cols: u16, rows: u16) -> Response { + if let Ok(process) = self.running_process(process_id) { + let session = process.main_session(); + if !session.terminal() { + return guest_error("invalid", "agent process has no terminal"); + } + self.process_runtime.block_on(session.resize( + u32::from(cols), + u32::from(rows), + 0, + 0, + )); + return Response::Resized; + } + let terminal = lock(&self.exec_handles) + .get(process_id) + .and_then(|handle| handle.terminal.clone()); + let Some(terminal) = terminal else { + return guest_error("invalid", "exec process has no terminal"); + }; + match self.process_runtime.block_on(terminal.resize(cols, rows)) { + Ok(()) => Response::Resized, + Err(error) => guest_error("failed", error.to_string()), + } + } + + fn connect_port( + &self, + target: LoopbackTarget, + ) -> Result { + let port_forward = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err("agent process has not been started".to_string()); + }; + process.port_forward() + }; + self.process_runtime + .block_on(port_forward.connect(target)) + .map_err(|error| error.to_string()) + } + + fn network_accept_context( + &self, + ) -> Result<(Arc, Arc), String> { + let process = loop { + let running = { + let state = lock(&self.state); + match &*state { + RuntimeState::Running(process) => Some(process.clone()), + _ => None, + } + }; + if let Some(process) = running { + break process; + } + std::thread::sleep(Duration::from_millis(10)); + }; + let listener = process + .network_listener() + .ok_or_else(|| "network mediation requested for a non-proxy policy".to_string())?; + Ok((listener, process)) + } + + #[cfg(test)] + fn accept_network(&self) -> Result<(tokio::net::TcpStream, BinaryIdentityWire), String> { + let (listener, process) = self.network_accept_context()?; + let (stream, workload_addr) = self + .process_runtime + .block_on(listener.accept()) + .map_err(|error| format!("accept boundary proxy connection: {error}"))?; + let proxy_addr = stream + .local_addr() + .map_err(|error| format!("read boundary proxy address: {error}"))?; + let identity = process + .identity_resolver() + .resolve_connection(workload_addr, proxy_addr); + Ok((stream, BinaryIdentityWire::from(identity))) + } + + fn stream_exec( + &self, + stream: ControlStream, + process_id: &str, + session: ExecSession, + ) -> Result<(), String> { + let process_id = process_id.to_string(); + let bridged = self.process_runtime.block_on(async move { + let stream = stream.into_tokio()?; + bridge_exec_stream(stream, session).await + }); + // Remove the handle even when the bridge fails; a failed stream + // must not leak its exec registration. + lock(&self.exec_handles).remove(&process_id); + bridged + } + + fn attach_process(&self, process_id: &str) -> Result<(MainAttachment, bool), Response> { + let process = self.running_process(process_id)?; + if process + .attached + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err(guest_error( + "denied", + "main process already has a control attachment", + )); + } + let session = process.main_session(); + let terminal = session.terminal(); + Ok((MainAttachment { process, session }, terminal)) + } + + fn stream_process( + &self, + stream: ControlStream, + attachment: MainAttachment, + ) -> Result<(), String> { + self.process_runtime.block_on(async move { + let stream = stream.into_tokio()?; + bridge_main_stream(stream, attachment.session.clone()).await + }) + } + + fn attach(&self, policy: SandboxPolicyWire) -> Response { + let mut state = lock(&self.state); + match &*state { + RuntimeState::AwaitingAttach => { + let prepared = match PreparedBoundary::establish( + &self.process_runtime, + &policy.clone().into(), + ) { + Ok(prepared) => prepared, + Err(error) => return guest_error("failed", error), + }; + *lock(&self.attached_policy) = Some(policy); + *state = RuntimeState::Bound(prepared); + Response::Attached + } + RuntimeState::Bound(_) | RuntimeState::Ready(_) | RuntimeState::Running(_) => { + // Idempotent retry of the same attach; a different policy + // must not be silently coalesced onto the bound boundary. + if lock(&self.attached_policy).as_ref() == Some(&policy) { + Response::Attached + } else { + guest_error("denied", "attach policy does not match the bound boundary") + } + } + } + } + + fn confirm(&self) -> Response { + let mut state = lock(&self.state); + match &*state { + RuntimeState::Bound(prepared) => { + if let Err(error) = prepared.confirm(&self.process_runtime) { + return guest_error("failed", error); + } + *state = RuntimeState::Ready(prepared.clone()); + Response::Confirmed + } + RuntimeState::Ready(_) | RuntimeState::Running(_) => Response::Confirmed, + RuntimeState::AwaitingAttach => { + guest_error("invalid", "boundary must be attached before confirm") + } + } + } + + #[allow(clippy::too_many_arguments)] + fn start_agent( + &self, + sandbox_id: String, + spec: AgentSpecWire, + policy: SandboxPolicyWire, + ca_cert: Option>, + ca_bundle: Option>, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + ) -> Response { + let mut state = lock(&self.state); + let requested = StartedAgent { + sandbox_id: sandbox_id.clone(), + spec: spec.clone(), + policy: policy.clone(), + ca_cert: ca_cert.clone(), + ca_bundle: ca_bundle.clone(), + provider_env_revision, + provider_env: provider_env.clone(), + }; + if let RuntimeState::Running(process) = &*state { + return if lock(&self.started_agent) + .as_ref() + .is_some_and(|accepted| accepted.matches_replay(&requested)) + { + Response::Started { + process_id: process.process_id(), + provider_env_revision: process.provider_credentials.snapshot().revision, + } + } else { + guest_error( + "denied", + "start_agent inputs do not match the running boundary", + ) + }; + } + let RuntimeState::Ready(prepared) = &*state else { + return guest_error("invalid", "boundary must be confirmed before start_agent"); + }; + let ca_file_paths = match install_ca_material(ca_cert, ca_bundle) { + Ok(paths) => paths, + Err(error) => return guest_error("failed", error), + }; + let mut policy = policy.into(); + let driver_identity = match &self.config.agent_identity { + BoundaryAgentIdentity::Resolved { uid, gid } => DriverIdentity::Resolved { + uid: *uid, + gid: *gid, + }, + BoundaryAgentIdentity::OciUser { declaration } => DriverIdentity::OciUser { + declaration: declaration.clone(), + }, + BoundaryAgentIdentity::None => DriverIdentity::None, + }; + let resolved_identity = match resolve_process_identity(&mut policy, &driver_identity) { + Ok(identity) => identity, + Err(error) => return guest_error("failed", error.to_string()), + }; + let launch = ManagedProcessLaunch { + sandbox_id, + spec, + policy, + resolved_identity, + provider_env_revision, + provider_env, + ca_file_paths, + }; + let process = + match ManagedProcess::spawn(&self.process_runtime, launch, prepared.clone()) { + Ok(process) => Arc::new(process), + Err(error) => return guest_error("failed", error), + }; + let process_id = process.process_id(); + *lock(&self.started_agent) = Some(requested); + *state = RuntimeState::Running(process); + Response::Started { + process_id, + provider_env_revision, + } + } + + fn update_provider_environment( + &self, + expected_revision: u64, + revision: u64, + provider_env: std::collections::HashMap, + ) -> Response { + let process = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return guest_error( + "invalid", + "agent process must be running before provider environment updates", + ); + }; + process.clone() + }; + let revision = process + .provider_credentials + .compare_and_install_child_env_snapshot(expected_revision, revision, provider_env); + Response::ProviderEnvironmentUpdated { revision } + } + + fn wait(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.wait() { + Ok(status) => Response::Exited { status }, + Err(error) => guest_error("failed", error), + } + } + + fn signal(&self, process_id: &str, signal: SignalWire) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(signal) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("terminated", error), + } + } + + fn terminate(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(SignalWire::Kill) { + Ok(()) => Response::Terminated, + Err(_) if process.has_exited() => Response::Terminated, + Err(error) => guest_error("failed", error), + } + } + + fn running_process(&self, process_id: &str) -> Result, Response> { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + if process.process_id() != process_id { + return Err(guest_error("invalid", "unknown process ID")); + } + Ok(process.clone()) + } + } + + impl PreparedBoundary { + fn establish( + runtime: &tokio::runtime::Handle, + policy: &openshell_core::policy::SandboxPolicy, + ) -> Result { + let netns = create_conformant_netns_for_proxy(policy) + .map_err(|error| format!("establish boundary workload network namespace: {error}"))? + .map(Arc::new); + let proxy_port = policy + .network + .proxy + .as_ref() + .and_then(|proxy| proxy.http_addr) + .map_or(3128, |address| address.port()); + let network_listener = if let Some(netns) = netns.as_ref() { + let address = std::net::SocketAddr::new(netns.host_ip(), proxy_port); + Some(Arc::new( + runtime + .block_on(tokio::net::TcpListener::bind(address)) + .map_err(|error| { + format!("bind boundary mediation listener {address}: {error}") + })?, + )) + } else { + None + }; + Ok(Self { + netns, + network_listener, + proxy_port, + }) + } + + fn confirm(&self, runtime: &tokio::runtime::Handle) -> Result<(), String> { + if let Some(netns) = self.netns.as_ref() { + runtime + .block_on( + netns + .egress_ceiling_verifier() + .verify_bounded(self.proxy_port, Duration::from_secs(2)), + ) + .map_err(|error| format!("verify boundary egress ceiling: {error}"))?; + if self.network_listener.is_none() { + return Err("boundary proxy namespace has no mediation listener".to_string()); + } + } + Ok(()) + } + } + + fn install_ca_material( + ca_cert: Option>, + ca_bundle: Option>, + ) -> Result, String> { + let (ca_cert, ca_bundle) = match (ca_cert, ca_bundle) { + (Some(ca_cert), Some(ca_bundle)) => (ca_cert, ca_bundle), + (None, None) => return Ok(None), + _ => { + return Err( + "boundary proxy CA certificate and bundle must be supplied together" + .to_string(), + ); + } + }; + install_ca_material_at(Path::new("/run/openshell-proxy-ca"), &ca_cert, &ca_bundle) + } + + fn install_ca_material_at( + directory: &Path, + ca_cert: &[u8], + ca_bundle: &[u8], + ) -> Result, String> { + use std::io::Write as _; + use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _}; + + let parent = directory + .parent() + .ok_or_else(|| "boundary proxy CA directory has no parent".to_string())?; + for path in [parent, directory] { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "boundary proxy CA directory component is a symlink: {}", + path.display() + )); + } + Ok(metadata) if !metadata.is_dir() => { + return Err(format!( + "boundary proxy CA directory component is not a directory: {}", + path.display() + )); + } + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => { + std::fs::create_dir(path).map_err(|error| { + format!( + "create boundary proxy CA directory {}: {error}", + path.display() + ) + })?; + } + Err(error) => { + return Err(format!( + "inspect boundary proxy CA directory {}: {error}", + path.display() + )); + } + } + let current_mode = std::fs::metadata(path) + .map_err(|error| { + format!( + "inspect boundary proxy CA directory permissions {}: {error}", + path.display() + ) + })? + .permissions() + .mode(); + if path == directory { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).map_err( + |error| { + format!( + "set boundary proxy CA directory permissions {}: {error}", + path.display() + ) + }, + )?; + } else if current_mode & 0o111 != 0o111 { + return Err(format!( + "boundary proxy CA parent is not traversable by workload identities: {}", + path.display() + )); + } + } + let ca_path = directory.join("ca.crt"); + let bundle_path = directory.join("ca-bundle.crt"); + for (path, contents, label) in [ + (&ca_path, ca_cert, "boundary proxy CA"), + (&bundle_path, ca_bundle, "boundary proxy CA bundle"), + ] { + let temporary = path.with_extension("tmp"); + if let Ok(metadata) = std::fs::symlink_metadata(&temporary) { + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(format!( + "refusing unsafe temporary {label} path: {}", + temporary.display() + )); + } + std::fs::remove_file(&temporary) + .map_err(|error| format!("remove stale temporary {label}: {error}"))?; + } + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o444) + .custom_flags(libc::O_NOFOLLOW) + .open(&temporary) + .map_err(|error| format!("create temporary {label}: {error}"))?; + if let Err(error) = file + .write_all(contents) + .and_then(|()| file.sync_all()) + .and_then(|()| file.set_permissions(std::fs::Permissions::from_mode(0o444))) + .and_then(|()| std::fs::rename(&temporary, path)) + { + let _ = std::fs::remove_file(&temporary); + return Err(format!("install {label}: {error}")); + } + } + Ok(Some((ca_path, bundle_path))) + } + + type ProcessExit = Result; + type SharedProcessExit = Arc<(Mutex>, Condvar)>; + + struct ManagedProcess { + pid: i32, + signaler: AgentSignaler, + exit: SharedProcessExit, + boundary_exec: Arc, + port_forward: Arc, + network_listener: Option>, + identity_resolver: ProcfsIdentityResolver, + main_session: Arc, + attached: AtomicBool, + _netns: Option>, + boundary_runtime: Arc, + provider_credentials: ProviderCredentialState, + } + + struct ManagedProcessLaunch { + sandbox_id: String, + spec: AgentSpecWire, + policy: openshell_core::policy::SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + } + + impl ManagedProcess { + fn spawn( + runtime: &tokio::runtime::Handle, + launch: ManagedProcessLaunch, + prepared: PreparedBoundary, + ) -> Result { + let ManagedProcessLaunch { + sandbox_id, + spec, + policy, + resolved_identity, + provider_env_revision, + provider_env, + ca_file_paths, + } = launch; + if spec.program.is_empty() { + return Err("agent program must not be empty".to_string()); + } + let boundary_runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); + let entrypoint_pid = Arc::new(AtomicU32::new(0)); + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + provider_env_revision, + provider_env.clone(), + ); + let mut spawned = runtime + .block_on(spawn_workload( + &spec.program, + &spec.args, + spec.workdir.as_deref(), + spec.timeout_secs, + spec.interactive, + Some(&sandbox_id), + None, + None, + false, + &policy, + resolved_identity, + ProcessEnforcementMode::Full, + entrypoint_pid.clone(), + None, + provider_credentials.clone(), + provider_env, + ca_file_paths, + AgentProposals::default(), + prepared.netns.as_deref(), + None, + None, + Some(boundary_runtime.clone()), + None, + )) + .map_err(|error| format!("start process supervisor leaf: {error:?}"))?; + let pid = i32::try_from(spawned.pid()) + .map_err(|_| "process supervisor PID does not fit i32".to_string())?; + let signaler = spawned.signaler(); + let boundary_exec = spawned.boundary_exec(); + let port_forward = spawned.port_forward(); + let main_session = spawned.main_session(); + let network_listener = prepared.network_listener.clone(); + let exit = Arc::new((Mutex::new(None), Condvar::new())); + let reaper_exit = exit.clone(); + runtime.spawn(async move { + let result = spawned + .wait() + .await + .map(process_status) + .map_err(|error| format!("wait for process supervisor leaf: {error}")); + let (state, changed) = &*reaper_exit; + *lock(state) = Some(result); + changed.notify_all(); + }); + Ok(Self { + pid, + signaler, + exit, + boundary_exec, + port_forward, + network_listener, + identity_resolver: ProcfsIdentityResolver { entrypoint_pid }, + main_session, + attached: AtomicBool::new(false), + _netns: prepared.netns, + boundary_runtime, + provider_credentials, + }) + } + + fn process_id(&self) -> String { + self.pid.to_string() + } + + fn wait(&self) -> ProcessExit { + let (state, changed) = &*self.exit; + let mut exit = lock(state); + while exit.is_none() { + exit = changed + .wait(exit) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + exit.as_ref().expect("exit checked above").clone() + } + + fn signal(&self, signal: SignalWire) -> Result<(), String> { + if self.has_exited() { + return Err("agent process has already exited".to_string()); + } + let result = match signal { + SignalWire::Term => self.signaler.term(), + SignalWire::Kill => self.signaler.kill(), + SignalWire::Int => self.signaler.interrupt(), + SignalWire::Hup => self.signaler.hangup(), + }; + result.map_err(|error| format!("signal process supervisor group: {error}")) + } + + fn has_exited(&self) -> bool { + let (state, _) = &*self.exit; + lock(state).is_some() + } + + fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } + + fn network_listener(&self) -> Option> { + self.network_listener.clone() + } + + fn identity_resolver(&self) -> ProcfsIdentityResolver { + self.identity_resolver.clone() + } + + fn main_session(&self) -> Arc { + self.main_session.clone() + } + } + + impl Drop for ManagedProcess { + fn drop(&mut self) { + self.boundary_runtime.deactivate(); + } + } + + async fn bridge_main_stream( + stream: openshell_isolation_interface::contract::BoundaryDuplexStream, + session: Arc, + ) -> Result<(), String> { + let (mut reader, writer) = tokio::io::split(stream); + let writer = Arc::new(tokio::sync::Mutex::new(writer)); + let (owner, input) = session.acquire_input().map_err(str::to_string)?; + let mut output = session.subscribe(); + let mut input_task = tokio::spawn(async move { + let mut input = Some(input); + while let Some((channel, payload)) = read_stream_frame(&mut reader).await? { + match channel { + STREAM_STDIN => { + let Some(input) = input.as_ref() else { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "main process stdin already closed", + )); + }; + input.send(payload).await.map_err(|_| { + io::Error::new(io::ErrorKind::BrokenPipe, "main process stdin closed") + })?; + } + // Keep reading after stdin closes so transport EOF still + // releases this control process's attachment lease. + STREAM_STDIN_CLOSED => drop(input.take()), + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected host-to-boundary main stream channel", + )); + } + } + } + Ok::<(), io::Error>(()) + }); + let result = loop { + let output_message = tokio::select! { + input_result = &mut input_task => { + break match input_result { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(format!("read main process attachment: {error}")), + Err(error) => Err(format!("join main process input stream: {error}")), + }; + } + output_message = output.recv() => output_message, + }; + let (channel, payload) = match output_message { + Ok(MainOutput::Stdout(payload)) => (STREAM_STDOUT, payload.to_vec()), + Ok(MainOutput::Stderr(payload)) => (STREAM_STDERR, payload.to_vec()), + Ok(MainOutput::Exit(code)) => { + let status = serde_json::to_vec(&ExitStatusWire::Exited(code)) + .map_err(|error| format!("encode main process exit: {error}"))?; + break write_stream_frame(&mut *writer.lock().await, STREAM_EXIT, &status) + .await + .map_err(|error| format!("write main process exit: {error}")); + } + Err(error) => { + break Err(format!( + "main process attachment lagged by {} chunks", + error.skipped + )); + } + }; + if let Err(error) = + write_stream_frame(&mut *writer.lock().await, channel, &payload).await + { + break Err(format!("write main process output: {error}")); + } + }; + input_task.abort(); + session.release_input(owner); + result + } + + fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn process_status(status: ProcessStatus) -> ExitStatusWire { + status.signal().map_or_else( + || ExitStatusWire::Exited(status.code()), + ExitStatusWire::Signaled, + ) + } + + fn guest_error(kind: &str, message: impl Into) -> Response { + Response::Error { + kind: kind.to_string(), + message: message.into(), + } + } + + fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + let max_len = left.len().max(right.len()); + let mut difference = left.len() ^ right.len(); + for index in 0..max_len { + let left_byte = left.get(index).copied().unwrap_or_default(); + let right_byte = right.get(index).copied().unwrap_or_default(); + difference |= usize::from(left_byte ^ right_byte); + } + difference == 0 + } + + enum ControlListener { + Vsock(OwnedFd), + Unix(std::os::unix::net::UnixListener), + Tcp(std::net::TcpListener), + TlsTcp { + listener: std::net::TcpListener, + server_config: Arc, + }, + } + + impl ControlListener { + fn bind(config: &BoundaryListenerConfig) -> io::Result { + match config { + BoundaryListenerConfig::Vsock { control_port } => { + Self::bind_vsock(*control_port).map(Self::Vsock) + } + BoundaryListenerConfig::Unix { socket_path } => { + let listener = std::os::unix::net::UnixListener::bind(socket_path)?; + // The driver-owned parent directory limits discovery and + // reachability; the protocol bootstrap token authenticates + // every request. Cross-UID placements such as rootful Docker + // need write permission on the socket inode itself. + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o666))?; + listener.set_nonblocking(true)?; + Ok(Self::Unix(listener)) + } + BoundaryListenerConfig::Tcp { address } => { + let listener = std::net::TcpListener::bind(address)?; + listener.set_nonblocking(true)?; + Ok(Self::Tcp(listener)) + } + BoundaryListenerConfig::TlsTcp { + address, + certificate_chain_path, + private_key_path, + } => { + let listener = std::net::TcpListener::bind(address)?; + listener.set_nonblocking(true)?; + let server_config = + load_tls_server_config(certificate_chain_path, private_key_path)?; + Ok(Self::TlsTcp { + listener, + server_config: Arc::new(server_config), + }) + } + } + } + + fn bind_vsock(port: u32) -> io::Result { + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "AF_VSOCK exceeds sa_family_t") + })?; + let address_length = libc::socklen_t::try_from(size_of::()) + .map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "sockaddr_vm exceeds socklen_t") + })?; + let raw_fd = unsafe { + libc::socket( + libc::AF_VSOCK, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK, + 0, + ) + }; + if raw_fd < 0 { + return Err(io::Error::last_os_error()); + } + let fd = unsafe { OwnedFd::from_raw_fd(raw_fd) }; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: port, + svm_cid: libc::VMADDR_CID_ANY, + svm_zero: [0; 4], + }; + let result = unsafe { + libc::bind( + fd.as_raw_fd(), + (&raw const address).cast::(), + address_length, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + if unsafe { libc::listen(fd.as_raw_fd(), 16) } < 0 { + return Err(io::Error::last_os_error()); + } + Ok(fd) + } + + #[cfg(test)] + fn tcp_local_addr(&self) -> io::Result { + match self { + Self::Tcp(listener) | Self::TlsTcp { listener, .. } => listener.local_addr(), + Self::Unix(_) | Self::Vsock(_) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "control listener is not TCP", + )), + } + } + + fn accept(&self) -> io::Result { + match self { + Self::Vsock(fd) => { + let raw_fd = unsafe { + libc::accept4( + fd.as_raw_fd(), + std::ptr::null_mut(), + std::ptr::null_mut(), + libc::SOCK_CLOEXEC, + ) + }; + if raw_fd < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(ControlStream::Vsock(unsafe { File::from_raw_fd(raw_fd) })) + } + } + Self::Unix(listener) => { + let (stream, _) = listener.accept()?; + Ok(ControlStream::Unix(stream)) + } + Self::Tcp(listener) => { + let (stream, _) = listener.accept()?; + if let Err(error) = stream.set_nodelay(true) { + tracing::debug!(%error, "Failed to set boundary TCP_NODELAY"); + } + Ok(ControlStream::Tcp(stream)) + } + Self::TlsTcp { + listener, + server_config, + } => { + let (stream, _) = listener.accept()?; + Ok(ControlStream::PendingTlsTcp { + stream, + server_config: server_config.clone(), + }) + } + } + } + } + + fn load_tls_server_config( + certificate_chain_path: &Path, + private_key_path: &Path, + ) -> io::Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let certificate_bytes = std::fs::read(certificate_chain_path)?; + let certificates = rustls_pemfile::certs(&mut certificate_bytes.as_slice()) + .collect::, _>>()?; + if certificates.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "boundary TLS certificate chain contains no certificates", + )); + } + let private_key_bytes = std::fs::read(private_key_path)?; + let private_key = rustls_pemfile::private_key(&mut private_key_bytes.as_slice())? + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "boundary TLS private-key file contains no private key", + ) + })?; + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, private_key) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error)) + } + + enum ControlStream { + Vsock(File), + Unix(std::os::unix::net::UnixStream), + Tcp(std::net::TcpStream), + PendingTlsTcp { + stream: std::net::TcpStream, + server_config: Arc, + }, + TlsTcp { + stream: Option>>, + runtime: tokio::runtime::Handle, + }, + } + + impl ControlStream { + fn establish(self, runtime: &tokio::runtime::Handle) -> io::Result { + let Self::PendingTlsTcp { + stream, + server_config, + } = self + else { + return Ok(self); + }; + stream.set_nonblocking(true)?; + let stream = { + let _guard = runtime.enter(); + tokio::net::TcpStream::from_std(stream)? + }; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + let acceptor = tokio_rustls::TlsAcceptor::from(server_config); + let stream = runtime.block_on(async { + tokio::time::timeout(CONTROL_IO_TIMEOUT, acceptor.accept(stream)) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS handshake timed out") + })? + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) + })?; + Ok(Self::TlsTcp { + stream: Some(Box::new(stream)), + runtime: runtime.clone(), + }) + } + + fn set_timeout(&self, timeout: Duration) -> io::Result<()> { + if let Self::Unix(stream) = self { + stream.set_read_timeout(Some(timeout))?; + return stream.set_write_timeout(Some(timeout)); + } + if let Self::Tcp(stream) = self { + stream.set_read_timeout(Some(timeout))?; + return stream.set_write_timeout(Some(timeout)); + } + if matches!(self, Self::TlsTcp { .. }) { + return Ok(()); + } + if matches!(self, Self::PendingTlsTcp { .. }) { + return Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )); + } + let Self::Vsock(file) = self else { + unreachable!("Unix and TCP streams returned above") + }; + let option_length = + libc::socklen_t::try_from(size_of::()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "timeval exceeds socklen_t") + })?; + let timeout_seconds = timeout.as_secs().try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "control timeout exceeds time_t", + ) + })?; + let timeout = libc::timeval { + tv_sec: timeout_seconds, + tv_usec: timeout.subsec_micros().into(), + }; + for option in [libc::SO_RCVTIMEO, libc::SO_SNDTIMEO] { + let result = unsafe { + libc::setsockopt( + file.as_raw_fd(), + libc::SOL_SOCKET, + option, + (&raw const timeout).cast(), + option_length, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) + } + + fn into_tokio( + self, + ) -> Result { + match self { + Self::Unix(stream) => { + stream.set_nonblocking(true).map_err(|error| { + format!("set boundary Unix stream nonblocking: {error}") + })?; + let stream = tokio::net::UnixStream::from_std(stream).map_err(|error| { + format!("register boundary Unix stream with Tokio: {error}") + })?; + Ok(Box::new(stream)) + } + Self::Tcp(stream) => { + stream + .set_nonblocking(true) + .map_err(|error| format!("set boundary TCP stream nonblocking: {error}"))?; + let stream = tokio::net::TcpStream::from_std(stream).map_err(|error| { + format!("register boundary TCP stream with Tokio: {error}") + })?; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + Ok(Box::new(stream)) + } + Self::TlsTcp { mut stream, .. } => Ok(stream + .take() + .expect("boundary TLS stream can only be converted once")), + Self::PendingTlsTcp { .. } => { + Err("boundary TLS stream has not completed its handshake".to_string()) + } + Self::Vsock(file) => { + let stream = + unsafe { std::os::unix::net::UnixStream::from_raw_fd(file.into_raw_fd()) }; + stream.set_nonblocking(true).map_err(|error| { + format!("set boundary vsock stream nonblocking: {error}") + })?; + let stream = tokio::net::UnixStream::from_std(stream).map_err(|error| { + format!("register boundary vsock stream with Tokio: {error}") + })?; + Ok(Box::new(stream)) + } + } + } + } + + impl Read for ControlStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self { + Self::Vsock(stream) => stream.read(buffer), + Self::Unix(stream) => stream.read(buffer), + Self::Tcp(stream) => stream.read(buffer), + Self::TlsTcp { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .read(buffer), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS read timed out") + })? + }), + Self::PendingTlsTcp { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), + } + } + } + + impl Write for ControlStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + match self { + Self::Vsock(stream) => stream.write(buffer), + Self::Unix(stream) => stream.write(buffer), + Self::Tcp(stream) => stream.write(buffer), + Self::TlsTcp { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .write(buffer), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS write timed out") + })? + }), + Self::PendingTlsTcp { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Vsock(stream) => stream.flush(), + Self::Unix(stream) => stream.flush(), + Self::Tcp(stream) => stream.flush(), + Self::TlsTcp { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .flush(), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS flush timed out") + })? + }), + Self::PendingTlsTcp { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), + } + } + } + + #[cfg(test)] + mod tests { + use super::*; + use rcgen::generate_simple_self_signed; + + #[test] + fn boundary_config_debug_redacts_token() { + let config = BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "never-log-this-never-log-this".to_string(), + protect_config_file: false, + listener: BoundaryListenerConfig::Vsock { control_port: 5500 }, + resource_claims: std::collections::BTreeMap::new(), + agent_identity: BoundaryAgentIdentity::Resolved { + uid: 10_001, + gid: 10_001, + }, + trusted_runtime_root: std::path::PathBuf::from( + "/opt/openshell/bin/openshell-runtime", + ), + child_env: std::collections::HashMap::new(), + }; + let debug = format!("{config:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn installed_proxy_ca_is_readable_by_a_non_root_workload_identity() { + use std::os::unix::fs::PermissionsExt as _; + + let root = tempfile::tempdir().expect("temporary CA root"); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let directory = root.path().join("openshell-proxy-ca"); + let (ca_path, bundle_path) = install_ca_material_at( + &directory, + b"public test certificate", + b"public test bundle", + ) + .expect("install proxy CA") + .expect("CA paths"); + + assert_eq!( + std::fs::metadata(directory.parent().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o111, + 0o111, + "non-root workload identities must be able to traverse the full path" + ); + assert_eq!( + std::fs::metadata(&directory).unwrap().permissions().mode() & 0o777, + 0o755, + "non-root workload identities must be able to traverse the CA directory" + ); + for (path, expected) in [ + (&ca_path, b"public test certificate".as_slice()), + (&bundle_path, b"public test bundle".as_slice()), + ] { + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o444, + "public CA material must be readable by the workload" + ); + assert_eq!(std::fs::read(path).unwrap(), expected); + } + } + + #[test] + fn proxy_ca_install_rejects_a_symlinked_directory() { + let root = tempfile::tempdir().expect("temporary CA root"); + let target = root.path().join("target"); + std::fs::create_dir(&target).unwrap(); + let parent = root.path(); + std::os::unix::fs::symlink(&target, parent.join("openshell-proxy-ca")).unwrap(); + + let error = install_ca_material_at( + &parent.join("openshell-proxy-ca"), + b"certificate", + b"bundle", + ) + .expect_err("symlinked CA directory must fail closed"); + assert!(error.contains("symlink"), "unexpected error: {error}"); + } + + #[test] + fn constant_time_comparison_checks_length_and_content() { + assert!(constant_time_eq(b"same", b"same")); + assert!(!constant_time_eq(b"same", b"different")); + assert!(!constant_time_eq(b"same", b"sam")); + } + + #[test] + fn control_connection_slots_bound_unauthenticated_threads() { + let active = Arc::new(AtomicUsize::new(MAX_CONTROL_CONNECTIONS - 1)); + let slot = acquire_control_connection_slot(&active).expect("last available slot"); + assert!(acquire_control_connection_slot(&active).is_none()); + drop(slot); + assert_eq!(active.load(Ordering::Acquire), MAX_CONTROL_CONNECTIONS - 1); + } + + #[tokio::test] + async fn pending_network_accept_cancels_when_control_disconnects() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind workload listener"); + let (control, peer) = tokio::io::duplex(64); + drop(peer); + let mut control: openshell_isolation_interface::contract::BoundaryDuplexStream = + Box::new(control); + + let error = tokio::time::timeout( + Duration::from_secs(1), + accept_network_while_control_connected(&listener, &mut control), + ) + .await + .expect("disconnect cancels accept promptly") + .expect_err("disconnected control cannot retain AcceptNetwork"); + assert!(error.contains("control disconnected")); + } + + #[test] + fn unix_listener_allows_authenticated_cross_uid_control() { + use std::os::unix::fs::PermissionsExt as _; + + let directory = tempfile::tempdir().expect("temporary directory"); + let socket_path = directory.path().join("control.sock"); + let _listener = ControlListener::bind(&BoundaryListenerConfig::Unix { + socket_path: socket_path.clone(), + }) + .expect("bind Unix listener"); + let mode = socket_path + .metadata() + .expect("socket metadata") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o666); + } + + #[test] + fn empty_oci_user_is_resolved_with_the_policy_inside_the_boundary() { + let config = BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "a".repeat(64), + protect_config_file: false, + listener: BoundaryListenerConfig::Vsock { control_port: 5500 }, + resource_claims: std::collections::BTreeMap::new(), + agent_identity: BoundaryAgentIdentity::OciUser { + declaration: String::new(), + }, + trusted_runtime_root: std::path::PathBuf::from("/opt/openshell/runtime"), + child_env: std::collections::HashMap::new(), + }; + + validate_config(&config).unwrap(); + } + + #[test] + fn tls_listener_preserves_session_when_control_switches_to_async_streaming() { + let directory = tempfile::tempdir().expect("temporary directory"); + let certificate_path = directory.path().join("boundary.crt"); + let private_key_path = directory.path().join("boundary.key"); + let certified = generate_simple_self_signed(vec!["boundary.test".to_string()]) + .expect("generate test certificate"); + std::fs::write(&certificate_path, certified.cert.pem()) + .expect("write test certificate"); + std::fs::write(&private_key_path, certified.key_pair.serialize_pem()) + .expect("write test private key"); + let listener = ControlListener::bind(&BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:0".parse().expect("valid address"), + certificate_chain_path: certificate_path, + private_key_path, + }) + .expect("bind TLS listener"); + let address = listener.tcp_local_addr().expect("TLS listener address"); + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test runtime"); + let server_runtime = runtime.handle().clone(); + let server = std::thread::spawn(move || { + let mut stream = loop { + match listener.accept() { + Ok(stream) => break stream, + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + std::thread::yield_now(); + } + Err(error) => panic!("accept TLS stream: {error}"), + } + } + .establish(&server_runtime) + .expect("establish TLS stream"); + let mut first = [0_u8; 4]; + Read::read_exact(&mut stream, &mut first).expect("read blocking TLS phase"); + assert_eq!(&first, b"sync"); + Write::write_all(&mut stream, b"ack1").expect("write blocking TLS phase"); + server_runtime.block_on(async move { + let mut stream = stream.into_tokio().expect("convert negotiated TLS stream"); + let mut second = [0_u8; 5]; + stream + .read_exact(&mut second) + .await + .expect("read async TLS phase"); + assert_eq!(&second, b"async"); + stream + .write_all(b"ack2") + .await + .expect("write async TLS phase"); + }); + }); + + runtime.block_on(async { + let mut roots = rustls::RootCertStore::empty(); + roots + .add(certified.cert.der().clone()) + .expect("trust test certificate"); + let client_config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + let stream = tokio::net::TcpStream::connect(address) + .await + .expect("connect TLS listener"); + let server_name = rustls::pki_types::ServerName::try_from("boundary.test") + .expect("valid server name"); + let mut stream = tokio_rustls::TlsConnector::from(Arc::new(client_config)) + .connect(server_name, stream) + .await + .expect("verify TLS listener"); + stream.write_all(b"sync").await.expect("write first phase"); + let mut first_ack = [0_u8; 4]; + stream + .read_exact(&mut first_ack) + .await + .expect("read first acknowledgement"); + assert_eq!(&first_ack, b"ack1"); + stream + .write_all(b"async") + .await + .expect("write second phase"); + let mut second_ack = [0_u8; 4]; + stream + .read_exact(&mut second_ack) + .await + .expect("read second acknowledgement"); + assert_eq!(&second_ack, b"ack2"); + }); + server.join().expect("TLS boundary server thread"); + } + + #[test] + fn control_restart_replays_running_lifecycle_exactly_once() { + const CHILD_MARKER: &str = "OPENSHELL_TEST_BOUNDARY_RECONNECT_CHILD"; + if std::env::var_os(CHILD_MARKER).is_none() { + let status = std::process::Command::new( + std::env::current_exe().expect("current test executable"), + ) + .args([ + "--exact", + "boundary_server::linux::tests::control_restart_replays_running_lifecycle_exactly_once", + "--nocapture", + ]) + .env(CHILD_MARKER, "1") + .status() + .expect("run isolated reconnect test"); + assert!(status.success(), "isolated reconnect test failed"); + return; + } + + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test process runtime"); + let boundary = Arc::new(BoundaryRuntime::new( + BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-reconnect".to_string(), + bootstrap_token: "a".repeat(32), + protect_config_file: false, + listener: BoundaryListenerConfig::Tcp { + address: "127.0.0.1:5500".parse().expect("control address"), + }, + resource_claims: std::collections::BTreeMap::new(), + agent_identity: BoundaryAgentIdentity::Resolved { + uid: nix::unistd::Uid::current().as_raw(), + gid: nix::unistd::Gid::current().as_raw(), + }, + trusted_runtime_root: "/tmp".into(), + child_env: std::collections::HashMap::new(), + }, + process_runtime.handle().clone(), + )); + let policy = SandboxPolicyWire::from(openshell_core::policy::SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }); + let spec = AgentSpecWire { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + workdir: None, + timeout_secs: 60, + interactive: false, + }; + + assert_eq!(boundary.attach(policy.clone()), Response::Attached); + assert_eq!(boundary.confirm(), Response::Confirmed); + let start = || { + boundary.start_agent( + "sandbox-reconnect".to_string(), + spec.clone(), + policy.clone(), + None, + None, + 0, + std::collections::HashMap::new(), + ) + }; + let Response::Started { + process_id, + provider_env_revision: 0, + } = start() + else { + panic!("initial start did not succeed"); + }; + + let (first_attachment, _) = boundary + .attach_process(&process_id) + .expect("initial main-process attachment"); + assert!(boundary.attach_process(&process_id).is_err()); + let (boundary_stream, control_stream) = + std::os::unix::net::UnixStream::pair().expect("main attachment socket pair"); + let stream_boundary = boundary.clone(); + let stream_thread = std::thread::spawn(move || { + stream_boundary + .stream_process(ControlStream::Unix(boundary_stream), first_attachment) + }); + drop(control_stream); + stream_thread + .join() + .expect("join disconnected main attachment") + .expect("transport EOF cleanly ends main attachment"); + let (replacement_attachment, _) = boundary + .attach_process(&process_id) + .expect("replacement main-process attachment after disconnect"); + drop(replacement_attachment); + + assert_eq!(boundary.attach(policy.clone()), Response::Attached); + assert_eq!(boundary.confirm(), Response::Confirmed); + assert_eq!( + start(), + Response::Started { + process_id: process_id.clone(), + provider_env_revision: 0, + } + ); + + let mut changed_policy = policy.clone(); + changed_policy.version += 1; + assert!(matches!( + boundary.attach(changed_policy.clone()), + Response::Error { kind, .. } if kind == "denied" + )); + assert!(matches!( + boundary.start_agent( + "sandbox-reconnect".to_string(), + spec, + changed_policy, + None, + None, + 0, + std::collections::HashMap::new(), + ), + Response::Error { kind, .. } if kind == "denied" + )); + + let (_exec_id, mut exec) = boundary + .start_exec(ExecSpecWire { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), "printf reconnected".to_string()], + env: Vec::new(), + workdir: None, + pty: false, + }) + .expect("exec after reconnect"); + let mut output = String::new(); + process_runtime + .block_on(exec.stdout.read_to_string(&mut output)) + .expect("read reconnect exec output"); + assert_eq!(output, "reconnected"); + assert!(matches!( + process_runtime.block_on(exec.process.wait()), + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(0)) + )); + assert_eq!(boundary.terminate(&process_id), Response::Terminated); + } + + #[test] + fn canonical_exit_preserves_pending_network_accept_and_exec() { + const CHILD_MARKER: &str = "OPENSHELL_TEST_RETAINED_BOUNDARY_CHILD"; + if std::env::var_os(CHILD_MARKER).is_none() { + let status = std::process::Command::new( + std::env::current_exe().expect("current test executable"), + ) + .args([ + "--exact", + "boundary_server::linux::tests::canonical_exit_preserves_pending_network_accept_and_exec", + "--nocapture", + ]) + .env(CHILD_MARKER, "1") + .status() + .expect("run isolated retained-boundary test"); + assert!(status.success(), "isolated retained-boundary test failed"); + return; + } + + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test process runtime"); + let network_listener = process_runtime + .block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) + .expect("bind mediated network listener"); + let network_address = network_listener + .local_addr() + .expect("mediated network listener address"); + let policy = openshell_core::policy::SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy { + mode: openshell_core::policy::NetworkMode::Proxy, + proxy: Some(openshell_core::policy::ProxyPolicy { + http_addr: Some("127.0.0.1:3128".parse().expect("proxy address")), + }), + }, + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }; + let prepared = PreparedBoundary { + netns: None, + network_listener: Some(Arc::new(network_listener)), + proxy_port: 3128, + }; + let agent_spec = AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: None, + timeout_secs: 5, + interactive: false, + }; + let wire_policy = SandboxPolicyWire::from(policy.clone()); + let process = Arc::new( + ManagedProcess::spawn( + process_runtime.handle(), + ManagedProcessLaunch { + sandbox_id: "sandbox-retained".to_string(), + spec: agent_spec.clone(), + policy, + resolved_identity: ResolvedProcessIdentity::new( + Some(nix::unistd::Uid::current().as_raw()), + Some(nix::unistd::Gid::current().as_raw()), + ), + provider_env_revision: 0, + provider_env: std::collections::HashMap::new(), + ca_file_paths: None, + }, + prepared, + ) + .expect("spawn canonical process"), + ); + let boundary = Arc::new(BoundaryRuntime::new( + BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-retained".to_string(), + bootstrap_token: "a".repeat(32), + protect_config_file: false, + listener: BoundaryListenerConfig::Tcp { + address: "127.0.0.1:5500".parse().expect("control address"), + }, + resource_claims: std::collections::BTreeMap::new(), + agent_identity: BoundaryAgentIdentity::None, + trusted_runtime_root: "/tmp".into(), + child_env: std::collections::HashMap::new(), + }, + process_runtime.handle().clone(), + )); + *lock(&boundary.state) = RuntimeState::Running(process.clone()); + *lock(&boundary.attached_policy) = Some(wire_policy.clone()); + *lock(&boundary.started_agent) = Some(StartedAgent { + sandbox_id: "sandbox-retained".to_string(), + spec: agent_spec.clone(), + policy: wire_policy.clone(), + ca_cert: None, + ca_bundle: None, + provider_env_revision: 0, + provider_env: std::collections::HashMap::new(), + }); + + // A replacement control process replays the durable lifecycle and + // receives the original process rather than spawning another one. + assert_eq!(boundary.attach(wire_policy.clone()), Response::Attached); + assert_eq!(boundary.confirm(), Response::Confirmed); + assert_eq!( + boundary.start_agent( + "sandbox-retained".to_string(), + agent_spec.clone(), + wire_policy.clone(), + None, + None, + 0, + std::collections::HashMap::new(), + ), + Response::Started { + process_id: process.process_id(), + provider_env_revision: 0, + } + ); + + assert_eq!( + boundary.update_provider_environment( + 0, + 2, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "refreshed".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 2 } + ); + assert_eq!( + boundary.update_provider_environment( + 0, + 1, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "stale".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 2 } + ); + assert_eq!( + boundary.update_provider_environment(2, 1, std::collections::HashMap::new()), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a numerically smaller opaque revision must revoke the environment" + ); + assert_eq!( + boundary.update_provider_environment( + 2, + 3, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "out-of-order".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a stale expected revision must not overwrite current state" + ); + assert_eq!( + boundary.update_provider_environment(1, 1, std::collections::HashMap::new()), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a duplicate update must be idempotent" + ); + + assert_eq!(boundary.attach(wire_policy.clone()), Response::Attached); + assert_eq!(boundary.confirm(), Response::Confirmed); + assert_eq!( + boundary.start_agent( + "sandbox-retained".to_string(), + agent_spec, + wire_policy, + None, + None, + 99, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "replacement-control-snapshot".to_string(), + )]), + ), + Response::Started { + process_id: process.process_id(), + provider_env_revision: 1, + }, + "a replacement control must resume from the boundary's current revision" + ); + + let (cancelled_id, cancelled_session) = boundary + .start_exec(ExecSpecWire { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + env: Vec::new(), + workdir: None, + pty: false, + }) + .expect("start exec whose response is cancelled"); + boundary.cancel_exec_start(&cancelled_id, cancelled_session.process.clone()); + assert!( + !lock(&boundary.exec_handles).contains_key(&cancelled_id), + "a cancelled exec handoff must remove its process handle" + ); + + let accept_boundary = boundary; + let pending_accept = std::thread::spawn(move || accept_boundary.accept_network()); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !process.has_exited() && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(process.has_exited(), "canonical process did not exit"); + + let _workload_connection = + std::net::TcpStream::connect(network_address).expect("connect mediated listener"); + let (_accepted, _identity) = pending_accept + .join() + .expect("pending AcceptNetwork thread") + .expect("AcceptNetwork remains viable after canonical exit"); + + let mut session = process_runtime + .block_on( + process.boundary_exec().exec( + ExecSpecWire { + program: "/bin/sh".to_string(), + args: vec![ + "-c".to_string(), + "if [ -z \"${ROTATED_TOKEN+x}\" ]; then printf revoked; else printf 'unexpected:%s' \"$ROTATED_TOKEN\"; fi" + .to_string(), + ], + env: Vec::new(), + workdir: None, + pty: false, + } + .into(), + ), + ) + .expect("exec after canonical exit"); + let mut output = String::new(); + process_runtime + .block_on(session.stdout.read_to_string(&mut output)) + .expect("read retained exec output"); + assert_eq!( + output, "revoked", + "exec after canonical exit must use the latest reconciled provider snapshot" + ); + assert!(matches!( + process_runtime.block_on(session.process.wait()), + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(0)) + )); + } + } +} + +#[cfg(target_os = "linux")] +pub use linux::run_boundary; + +#[cfg(not(target_os = "linux"))] +pub fn run_boundary(_config_path: &Path) -> Result<(), String> { + Err("boundary mode is supported only on Linux".to_string()) +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index b2ba533515..77c85398e4 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -16,6 +16,7 @@ compile_error!( ); mod activity_aggregator; +mod boundary_server; mod denial_aggregator; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod google_cloud_metadata; @@ -26,6 +27,7 @@ mod sidecar_control; use miette::{IntoDiagnostic, Result, WrapErr}; use std::future::Future; +use std::io::Write as _; use std::pin::Pin; use std::sync::Arc; #[cfg(target_os = "linux")] @@ -68,6 +70,111 @@ use openshell_ocsf::{ /// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; +/// Run the authenticated boundary-local half of the supervisor. +/// +/// # Errors +/// +/// Returns an error when the protected boundary configuration is invalid or +/// the boundary listener cannot be established. +pub fn run_boundary(config_path: &std::path::Path) -> Result<()> { + boundary_server::run_boundary(config_path).map_err(|error| miette::miette!(error)) +} + +async fn retain_remote_access_plane( + proxy_exited: impl Future, + shutdown_requested: impl Future, +) -> Result<()> { + tokio::pin!(proxy_exited); + tokio::pin!(shutdown_requested); + tokio::select! { + () = &mut proxy_exited => Err(miette::miette!( + "control-mode proxy accept loop exited unexpectedly" + )), + () = &mut shutdown_requested => Ok(()), + } +} + +async fn completion_phase_or_shutdown(phase: F, mut shutdown: Pin<&mut S>) -> bool +where + F: Future, + S: Future + ?Sized, +{ + tokio::pin!(phase); + tokio::select! { + () = &mut phase => false, + () = &mut shutdown => true, + } +} + +struct ControlReadiness { + task: tokio::task::JoinHandle<()>, + #[cfg(test)] + address: std::net::SocketAddr, +} + +impl ControlReadiness { + async fn start(bind_ip: std::net::IpAddr, port: u16) -> Result { + Self::start_at(control_readiness_address(bind_ip, port)?).await + } + + async fn start_at(address: std::net::SocketAddr) -> Result { + let listener = tokio::net::TcpListener::bind(address) + .await + .into_diagnostic() + .wrap_err_with(|| format!("bind control-mode readiness listener on {address}"))?; + #[cfg(test)] + let address = listener.local_addr().into_diagnostic()?; + let task = tokio::spawn(async move { + loop { + match listener.accept().await { + Ok((stream, _)) => drop(stream), + Err(error) => { + tracing::warn!(%error, "control-mode readiness accept failed; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + }); + Ok(Self { + task, + #[cfg(test)] + address, + }) + } +} + +fn control_readiness_address(bind_ip: std::net::IpAddr, port: u16) -> Result { + if port == 0 { + return Err(miette::miette!( + "control-mode readiness port must be nonzero" + )); + } + Ok(std::net::SocketAddr::new(bind_ip, port)) +} + +impl Drop for ControlReadiness { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[cfg(unix)] +async fn wait_for_control_shutdown_signal() { + use tokio::signal::unix::{SignalKind, signal}; + + let mut sigterm = signal(SignalKind::terminate()).expect("install control SIGTERM handler"); + let mut sigint = signal(SignalKind::interrupt()).expect("install control SIGINT handler"); + tokio::select! { + _ = sigterm.recv() => {} + _ = sigint.recv() => {} + } +} + +#[cfg(not(unix))] +async fn wait_for_control_shutdown_signal() { + let _ = tokio::signal::ctrl_c().await; +} + use openshell_core::denial::DenialEvent; use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; use openshell_core::proposals::AgentProposals; @@ -78,7 +185,6 @@ use openshell_supervisor_process::process::ProcessEnforcementMode; pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; use openshell_supervisor_process::skills; use tokio::sync::mpsc::UnboundedSender; -#[cfg(any(test, target_os = "linux"))] use tokio::time::timeout; const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; @@ -97,6 +203,15 @@ fn has_network_runtime_capability(capabilities: Option<&str>, required: &str) -> const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; +fn shared_ssh_socket_from_env() -> bool { + std::env::var(openshell_core::sandbox_env::SSH_SOCKET_SHARED) + .is_ok_and(|value| shared_ssh_socket_value(&value)) +} + +fn shared_ssh_socket_value(value: &str) -> bool { + value == "1" || value.eq_ignore_ascii_case("true") +} + /// Run a command in the sandbox. /// /// # Errors @@ -120,13 +235,17 @@ pub async fn run_sandbox( policy_rules: Option, policy_data: Option, ssh_socket_path: Option, - _health_check: bool, - _health_port: u16, + health_check: bool, + health_port: u16, + health_bind_ip: std::net::IpAddr, inference_routes: Option, ocsf_enabled: Arc, network_enabled: bool, process_enabled: bool, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + topology_descriptor: Option, + admitted_isolation_backend: Option, + main_exit_marker: Option, ) -> Result { let (program, args) = command .split_first() @@ -204,7 +323,7 @@ pub async fn run_sandbox( MiddlewareRegistryStatus::Synchronized, loaded_policy_origin, bootstrap.agent_proposals_enabled, - false, + shared_ssh_socket_from_env(), ) } else { load_policy( @@ -221,9 +340,16 @@ pub async fn run_sandbox( // Normalize the active driver's identity contract once, while both the // policy and launched image filesystem are available. Kubernetes and // OpenShift retain their authoritative numeric pair; Docker fills only - // omitted policy fields from OCI Config.User. + // omitted policy fields from OCI Config.User. A remote boundary resolves + // identity in its own filesystem instead; control must not interpret + // guest account data against the host's /etc/passwd and /etc/group. #[cfg(unix)] - let (resolved_process_identity, workspace) = { + let (resolved_process_identity, workspace) = if topology_descriptor.is_some() { + ( + openshell_supervisor_process::process::ResolvedProcessIdentity::default(), + openshell_supervisor_process::process::ResolvedWorkspace::new(workdir.clone(), false), + ) + } else { let driver_identity = openshell_supervisor_process::identity::DriverIdentity::from_env()?; let use_workdir_as_home = matches!( &driver_identity, @@ -395,24 +521,78 @@ pub async fn run_sandbox( // the entrypoint process's /proc/net/tcp for identity binding. let entrypoint_pid = Arc::new(AtomicU32::new(0)); + // A separated topology uses the shared authenticated boundary protocol. + // The admitted backend name is resolved independently of the protected + // descriptor, and generic supervisor code never imports a driver crate. + let remote_boundary = if let Some(descriptor) = topology_descriptor { + if sidecar_network_enforcement || !network_enabled || !process_enabled { + return Err(miette::miette!( + "--mode=control requires combined network and process supervision" + )); + } + let admitted_backend_name = admitted_isolation_backend.ok_or_else(|| { + miette::miette!("topology descriptor supplied without an admitted isolation backend") + })?; + let ca_file_paths = Arc::new(std::sync::Mutex::new(None)); + let backend: Arc = Arc::new( + openshell_isolation_interface::remote::RemoteIsolationBackend::new( + admitted_backend_name.clone(), + ca_file_paths.clone(), + provider_credentials.clone(), + ), + ); + let mut registry = openshell_isolation_interface::contract::BackendRegistry::new(); + registry + .register(backend) + .map_err(|error| miette::miette!(error.to_string()))?; + let (backend, verified) = registry + .resolve(descriptor, &admitted_backend_name) + .map_err(|error| miette::miette!(error.to_string()))?; + let context = openshell_isolation_interface::contract::SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + policy: policy.clone(), + agent: openshell_isolation_interface::AgentSpec { + program: program.clone(), + args: args.to_vec(), + workdir: workspace.owned_root(), + timeout_secs, + interactive, + }, + }; + let bound = backend + .attach(verified, context) + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %admitted_backend_name, "Isolation boundary attached"); + Some((bound, admitted_backend_name, ca_file_paths)) + } else { + if admitted_isolation_backend.is_some() { + return Err(miette::miette!( + "admitted isolation backend supplied without a topology descriptor" + )); + } + None + }; + // Create the workload's network namespace. It is shared infrastructure: // the proxy binds to its host-side veth IP, the bypass monitor reads // /dev/kmsg from inside it, and the workload child / SSH sessions enter // it via setns(). The RAII handle lives in this frame for the duration // of the sandbox. #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { + let netns = if remote_boundary.is_none() && network_enabled && !sidecar_network_enforcement { openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? } else { None }; #[cfg(target_os = "linux")] - let transparent_tcp_requested = opa_engine - .as_ref() - .map(|engine| engine.policy_dns_eligibility_snapshot()) - .transpose()? - .is_some_and(|snapshot| !snapshot.endpoints.is_empty()); + let transparent_tcp_requested = remote_boundary.is_none() + && opa_engine + .as_ref() + .map(|engine| engine.policy_dns_eligibility_snapshot()) + .transpose()? + .is_some_and(|snapshot| !snapshot.endpoints.is_empty()); #[cfg(target_os = "linux")] let runtime_capabilities = std::env::var(openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES).ok(); @@ -527,6 +707,16 @@ pub async fn run_sandbox( // API read the current value so proposals target the correct workspace. let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); + let remote_network_source = remote_boundary + .as_ref() + .map(|(bound, _, _)| bound.network_mediation_source()); + let remote_dns_source = remote_boundary + .as_ref() + .and_then(|(bound, _, _)| bound.dns_mediation_source()); + let remote_host_gateway_ip = remote_boundary + .as_ref() + .and_then(|(bound, _, _)| bound.host_gateway_ip()); + let mut networking = if network_enabled { #[cfg(target_os = "linux")] let proxy_bind_ip = netns @@ -553,9 +743,11 @@ pub async fn run_sandbox( agent_proposals.clone(), workspace_rx.clone(), &upstream_proxy_args, - None, + remote_host_gateway_ip, #[cfg(target_os = "linux")] transparent_runtime, + remote_network_source, + remote_dns_source, ) .await?, ) @@ -563,6 +755,25 @@ pub async fn run_sandbox( None }; + let remote_ready = if let Some((bound, backend_name, ca_file_paths)) = remote_boundary { + ca_file_paths + .lock() + .map_err(|_| miette::miette!("boundary CA path lock is poisoned"))? + .clone_from( + &networking + .as_ref() + .and_then(|runtime| runtime.ca_file_paths.clone()), + ); + let ready = bound + .confirm() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %backend_name, "Isolation boundary enforcement confirmed"); + Some((ready, backend_name)) + } else { + None + }; + #[cfg(target_os = "linux")] let sidecar_control_server = if network_enabled && sidecar_network_enforcement { if !matches!(policy.network.mode, NetworkMode::Proxy) { @@ -850,7 +1061,136 @@ pub async fn run_sandbox( }; tokio::pin!(proxy_exited); - let exit_code = if process_enabled { + let exit_code = if let Some((ready, backend_name)) = remote_ready { + let running = ready + .start_agent() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %backend_name, "Isolation boundary agent started"); + let agent = running.agent(); + let boundary_access = openshell_supervisor_process::delegated::start_boundary_access( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + ssh_socket_path.as_deref(), + shared_ssh_socket_from_env(), + networking + .as_ref() + .and_then(|runtime| runtime.ca_file_paths.clone()), + process_enforcement_mode, + running.exec(), + running.port_forward(), + agent.clone(), + ) + .await?; + info!(backend = %backend_name, "Control-mode access plane started"); + let mut control_readiness = if health_check { + Some(ControlReadiness::start(health_bind_ip, health_port).await?) + } else { + None + }; + let instance_id = boundary_access.instance_id().to_string(); + let wait_agent = agent.clone(); + let shutdown_requested = wait_for_control_shutdown_signal(); + tokio::pin!(shutdown_requested); + let wait = async move { + wait_agent + .wait() + .await + .map(|status| match status { + openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code) => { + code + } + openshell_isolation_interface::contract::BoundaryExitStatus::Signaled( + signal, + ) => 128_i32.saturating_add(signal), + }) + .map_err(|error| miette::miette!(error.to_string())) + }; + let (exit_code, mut retain_access) = tokio::select! { + result = wait => (result?, true), + () = &mut proxy_exited => { + let _ = agent.terminate().await; + return Err(miette::miette!( + "control-mode proxy accept loop exited unexpectedly" + )); + } + () = &mut shutdown_requested => { + let _ = agent + .signal(openshell_isolation_interface::contract::BoundarySignal::Term) + .await; + let status = if let Ok(result) = timeout(Duration::from_secs(5), agent.wait()).await { + result + } else { + let _ = agent.terminate().await; + agent.wait().await + } + .map_err(|error| miette::miette!(error.to_string()))?; + let exit_code = match status { + openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code) => code, + openshell_isolation_interface::contract::BoundaryExitStatus::Signaled(signal) => { + 128_i32.saturating_add(signal) + } + }; + (exit_code, false) + } + }; + if !retain_access { + control_readiness.take(); + } + boundary_access + .publish_main_exit(exit_code, await_main_process_attachment) + .await; + // `shutdown_requested` has already completed when shutdown won the + // lifecycle select above and must not be polled again. + let mut completion_cancelled = !retain_access; + if retain_access && let Some(marker) = main_exit_marker.as_deref() { + persist_main_exit_marker(marker, exit_code) + .into_diagnostic() + .wrap_err("persist canonical-process completion marker")?; + } + if !completion_cancelled + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + { + let report = openshell_supervisor_process::delegated::report_main_process_exit( + endpoint, + id, + &instance_id, + exit_code, + ); + completion_cancelled = + completion_phase_or_shutdown(report, shutdown_requested.as_mut()).await; + } + if !completion_cancelled { + let drain = boundary_access.drain_main_terminal_delivery(); + completion_cancelled = + completion_phase_or_shutdown(drain, shutdown_requested.as_mut()).await; + } + if !completion_cancelled + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + { + let finalize = openshell_supervisor_process::delegated::finalize_main_process_exit( + endpoint, + id, + &instance_id, + ); + completion_cancelled = + completion_phase_or_shutdown(finalize, shutdown_requested.as_mut()).await; + } + if completion_cancelled { + retain_access = false; + control_readiness.take(); + } + if retain_access { + info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); + retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; + } + drop(control_readiness); + drop(running); + drop(boundary_access); + exit_code + } else if process_enabled { let ca_file_paths = networking .as_ref() .and_then(|n| n.ca_file_paths.clone()) @@ -1144,6 +1484,41 @@ pub async fn run_sandbox( Ok(exit_code) } +fn persist_main_exit_marker(path: &std::path::Path, exit_code: i32) -> std::io::Result<()> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("completion marker has no parent: {}", path.display()), + ) + })?; + let name = path.file_name().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("completion marker has no file name: {}", path.display()), + ) + })?; + let temporary = parent.join(format!( + ".{}.tmp-{}", + name.to_string_lossy(), + std::process::id() + )); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + let mut file = options.open(&temporary)?; + writeln!(file, "exit_code={exit_code}")?; + file.sync_all()?; + std::fs::rename(&temporary, path)?; + std::fs::File::open(parent)?.sync_all() +} + /// Wait for SIGINT or SIGTERM. Used in network-only mode where there is /// no entrypoint child whose lifetime drives the supervisor's exit. async fn wait_for_shutdown_signal() { @@ -1450,7 +1825,11 @@ fn spawn_sidecar_entrypoint_handler( endpoint.clone(), id.clone(), trusted_ssh_socket_path.clone(), - None, + Arc::new( + openshell_supervisor_process::boundary_io::NetnsPortForward::new( + None, None, + ), + ), Some(supervisor_pid), Arc::clone(&terminating), started.instance_id.clone(), @@ -4476,6 +4855,97 @@ fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String mod tests { use super::*; + #[test] + fn shared_ssh_socket_setting_is_explicit() { + assert!(shared_ssh_socket_value("1")); + assert!(shared_ssh_socket_value("true")); + assert!(shared_ssh_socket_value("TRUE")); + assert!(!shared_ssh_socket_value("0")); + assert!(!shared_ssh_socket_value("yes")); + } + + #[tokio::test] + async fn control_readiness_exists_only_while_guard_is_live() { + let readiness = ControlReadiness::start_at("127.0.0.1:0".parse().unwrap()) + .await + .expect("start readiness listener"); + let address = readiness.address; + tokio::net::TcpStream::connect(address) + .await + .expect("running control accepts readiness probes"); + + drop(readiness); + tokio::task::yield_now().await; + let result = timeout( + Duration::from_secs(1), + tokio::net::TcpStream::connect(address), + ) + .await + .expect("closed readiness listener fails promptly"); + assert!(result.is_err()); + } + + #[test] + fn control_readiness_preserves_explicit_ipv6_bind_address() { + let address = control_readiness_address("2001:db8::17".parse().unwrap(), 8_080) + .expect("valid IPv6 readiness address"); + assert_eq!(address, "[2001:db8::17]:8080".parse().unwrap()); + } + + #[test] + fn control_readiness_rejects_zero_port() { + let error = control_readiness_address(std::net::Ipv6Addr::UNSPECIFIED.into(), 0) + .expect_err("zero readiness port must be rejected"); + assert!(error.to_string().contains("must be nonzero")); + } + + #[test] + fn main_exit_marker_atomically_replaces_previous_value() { + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("main-exited"); + std::fs::write(&marker, b"stale\n").unwrap(); + + persist_main_exit_marker(&marker, 23).unwrap(); + + assert_eq!(std::fs::read_to_string(&marker).unwrap(), "exit_code=23\n"); + assert!( + !directory + .path() + .join(format!(".main-exited.tmp-{}", std::process::id())) + .exists() + ); + } + + #[tokio::test] + async fn remote_access_plane_outlives_main_completion_until_teardown() { + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let retained = retain_remote_access_plane(std::future::pending(), async { + let _ = shutdown_rx.await; + }); + tokio::pin!(retained); + + assert!( + timeout(Duration::from_millis(10), &mut retained) + .await + .is_err(), + "access plane must remain live after canonical process completion" + ); + shutdown_tx.send(()).expect("request teardown"); + timeout(Duration::from_secs(1), &mut retained) + .await + .expect("teardown should release retained access plane") + .expect("clean teardown"); + } + + #[tokio::test] + async fn completion_retry_phase_is_cancelled_by_shutdown() { + let mut shutdown = Box::pin(std::future::ready(())); + assert!( + completion_phase_or_shutdown(std::future::pending(), shutdown.as_mut()).await, + "shutdown must cancel an indefinitely retrying completion phase" + ); + } + #[test] fn transparent_tcp_capability_requires_exact_driver_marker() { let required = openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY; diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 7108378654..d0b2484179 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -55,17 +55,22 @@ const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; #[cfg(target_os = "linux")] const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; -/// Which supervisor leaves are enabled in this process. +/// Which supervisor role or legacy leaves are enabled in this process. /// -/// Parsed from a comma-separated `--mode` value, e.g. `network`, -/// `process`, or `network,process`. `network-init` is a one-shot setup mode -/// used by the Kubernetes sidecar topology and cannot be combined with other -/// mode components. At least one must be set. +/// `control` is the RFC 0012 logical supervisor outside the workload; +/// `boundary` is its authenticated boundary-local counterpart. Legacy leaf +/// combinations remain accepted while existing deployments migrate. #[derive(Clone, Copy, Debug)] +#[allow( + clippy::struct_excessive_bools, + reason = "legacy leaf combinations coexist with the two migration roles" +)] struct Mode { network: bool, process: bool, network_init: bool, + control: bool, + boundary: bool, } impl std::str::FromStr for Mode { @@ -76,25 +81,36 @@ impl std::str::FromStr for Mode { network: false, process: false, network_init: false, + control: false, + boundary: false, }; for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { match part { "network" => mode.network = true, "process" => mode.process = true, "network-init" => mode.network_init = true, + "control" => { + mode.control = true; + mode.network = true; + mode.process = true; + } + "boundary" => mode.boundary = true, other => { return Err(format!( - "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" + "unknown mode component '{other}' (expected 'control', 'boundary', 'network', 'process', or 'network-init')" )); } } } - if mode.network_init && (mode.network || mode.process) { - return Err("--mode=network-init cannot be combined with other components".into()); + if (mode.control || mode.boundary || mode.network_init) + && s.split(',').filter(|part| !part.trim().is_empty()).count() != 1 + { + return Err("control, boundary, and network-init modes cannot be combined".into()); } - if !mode.network && !mode.process && !mode.network_init { + if !mode.network && !mode.process && !mode.network_init && !mode.boundary { return Err( - "--mode must enable at least one of: network, process, network-init".into(), + "--mode must enable one of: control, boundary, network, process, network-init" + .into(), ); } Ok(mode) @@ -169,15 +185,27 @@ struct Args { #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] inference_routes: Option, - /// Enable health check endpoint. + /// Expose control-mode TCP readiness after the boundary is running. #[arg(long)] health_check: bool, - /// Port for health check endpoint. + /// TCP readiness port used with `--health-check`. #[arg(long, default_value = "8080")] health_port: u16, - /// Which supervisor components to run. Comma-separated list of + /// IP address for the control-mode TCP readiness listener. Kubernetes + /// IPv6 pods should inject `status.podIP` through `OPENSHELL_HEALTH_BIND_IP`. + #[arg( + long, + default_value = "0.0.0.0", + env = openshell_core::sandbox_env::HEALTH_BIND_IP + )] + health_bind_ip: std::net::IpAddr, + + /// Supervisor role to run. Drivers use `control` outside the workload and + /// `boundary` inside it. Legacy comma-separated leaf modes remain accepted. + /// + /// Legacy values are a comma-separated list of /// "network" and/or "process". Defaults to both (single-binary /// topology). Use --mode=network for a network-only sidecar, or /// --mode=process for a process-only supervisor when network @@ -186,6 +214,10 @@ struct Args { #[arg(long, default_value = DEFAULT_MODE)] mode: Mode, + /// Protected boundary configuration used only with `--mode=boundary`. + #[arg(long)] + boundary_config: Option, + /// UID that the long-running Kubernetes network sidecar will run as. /// `--mode=network-init` installs nftables rules that exempt this UID. #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] @@ -235,6 +267,44 @@ struct Args { /// re-signed upstream certificates and the sandbox trust bundle. #[arg(long)] upstream_proxy_ca_bundle: Option, + + /// Backend named by the compute driver's topology descriptor. + #[arg(long)] + topology_backend_name: Option, + + /// Isolation Backend interface version named by the topology descriptor. + #[arg(long)] + topology_version: Option, + + /// Base64-encoded opaque topology-descriptor payload. + #[arg(long, conflicts_with = "topology_payload_file")] + topology_payload_base64: Option, + + /// Driver-owned protected file holding the raw topology payload. + #[arg(long)] + topology_payload_file: Option, + + /// Driver-owned durable marker written after canonical process completion. + /// Used by separated control processes whose boundary remains alive. + #[arg(long, hide = true)] + main_exit_marker: Option, +} + +fn validate_main_exit_marker(mode: Mode, marker: Option<&Path>) -> Result<()> { + let Some(marker) = marker else { + return Ok(()); + }; + if !mode.control { + return Err(miette::miette!( + "--main-exit-marker is only valid with --mode=control" + )); + } + if !marker.is_absolute() { + return Err(miette::miette!( + "--main-exit-marker must be an absolute path" + )); + } + Ok(()) } /// Internal one-shot command used by the privileged supervisor to validate an @@ -541,6 +611,29 @@ fn main() -> Result<()> { } let args = Args::parse(); + validate_main_exit_marker(args.mode, args.main_exit_marker.as_deref())?; + + if args.mode.boundary { + let console_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); + let _ = tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .try_init(); + let config = args + .boundary_config + .as_deref() + .ok_or_else(|| miette::miette!("--mode=boundary requires --boundary-config"))?; + return openshell_sandbox::run_boundary(config); + } + if args.boundary_config.is_some() { + return Err(miette::miette!( + "--boundary-config is only valid with --mode=boundary" + )); + } if args.mode.network_init { let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); @@ -692,6 +785,53 @@ fn main() -> Result<()> { proxy_ca_bundle: args.upstream_proxy_ca_bundle, }; + let topology_payload = match (args.topology_payload_base64, args.topology_payload_file) { + (None, None) => None, + (Some(encoded), None) => { + use base64::Engine as _; + Some( + base64::engine::general_purpose::STANDARD + .decode(encoded) + .into_diagnostic()?, + ) + } + (None, Some(path)) => Some(std::fs::read(&path).map_err(|error| { + miette::miette!("read topology payload {}: {error}", path.display()) + })?), + (Some(_), Some(_)) => unreachable!("clap rejects conflicting payload flags"), + }; + let topology_descriptor = match ( + args.topology_backend_name, + args.topology_version, + topology_payload, + ) { + (None, None, None) => None, + (Some(backend_name), Some(version), Some(payload)) => Some( + openshell_isolation_interface::contract::TopologyDescriptor { + backend_name, + version, + payload, + }, + ), + _ => { + return Err(miette::miette!( + "topology descriptor requires backend name, version, and payload" + )); + } + }; + let admitted_isolation_backend = + std::env::var(openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND).ok(); + if args.mode.control && topology_descriptor.is_none() { + return Err(miette::miette!( + "--mode=control requires a protected topology descriptor" + )); + } + if topology_descriptor.is_some() && !args.mode.control { + return Err(miette::miette!( + "a topology descriptor requires --mode=control" + )); + } + run_sandbox( command, workdir, @@ -706,11 +846,15 @@ fn main() -> Result<()> { args.ssh_socket_path, args.health_check, args.health_port, + args.health_bind_ip, args.inference_routes, ocsf_enabled, args.mode.network, args.mode.process, upstream_proxy_args, + topology_descriptor, + admitted_isolation_backend, + args.main_exit_marker, ) .await })?; @@ -815,10 +959,43 @@ mod tests { assert!(err.contains("cannot be combined")); } + #[test] + fn mode_parses_control_as_combined_supervisor() { + let mode = "control".parse::().unwrap(); + assert!(mode.control); + assert!(mode.network); + assert!(mode.process); + assert!(!mode.boundary); + } + + #[test] + fn mode_parses_boundary_as_standalone_leaf() { + let mode = "boundary".parse::().unwrap(); + assert!(mode.boundary); + assert!(!mode.control); + assert!(!mode.network); + assert!(!mode.process); + } + + #[test] + fn mode_rejects_control_or_boundary_combinations() { + assert!("control,network".parse::().is_err()); + assert!("boundary,process".parse::().is_err()); + } + #[test] fn mode_rejects_empty_value() { let err = "".parse::().unwrap_err(); - assert!(err.contains("at least one")); + assert!(err.contains("--mode must enable one of")); + } + + #[test] + fn main_exit_marker_requires_control_mode_and_absolute_path() { + let control = "control".parse::().unwrap(); + let combined = DEFAULT_MODE.parse::().unwrap(); + assert!(validate_main_exit_marker(control, Some(Path::new("/state/exited"))).is_ok()); + assert!(validate_main_exit_marker(control, Some(Path::new("relative"))).is_err()); + assert!(validate_main_exit_marker(combined, Some(Path::new("/state/exited"))).is_err()); } #[cfg(target_os = "linux")] diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index d4d08da251..760e54fc0e 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -2196,10 +2196,7 @@ fn sandbox_relay_reachable(state: &ServerState, sandbox: &Sandbox) -> bool { let phase = SandboxPhase::try_from(sandbox.phase()).ok(); matches!(phase, Some(SandboxPhase::Ready)) || (matches!(phase, Some(SandboxPhase::Completed | SandboxPhase::Error)) - && state.supervisor_sessions.has_session(sandbox.object_id()) - && !state - .supervisor_sessions - .terminal_delivery_finalized(sandbox.object_id())) + && state.supervisor_sessions.has_session(sandbox.object_id())) } pub(super) async fn handle_create_ssh_session( @@ -5465,6 +5462,9 @@ mod tests { .supervisor_sessions .finalize_main_process_exit("sandbox-work") ); + assert!(sandbox_relay_reachable(&state, &sandbox)); + + assert!(state.supervisor_sessions.disconnect("sandbox-work")); assert!(!sandbox_relay_reachable(&state, &sandbox)); } @@ -6228,7 +6228,16 @@ mod tests { let mut sandbox = test_sandbox("cross-ws", Vec::new()); sandbox.metadata.as_mut().unwrap().workspace = "other-workspace".to_string(); + sandbox.set_phase(SandboxPhase::Completed as i32); state.store.put_message(&sandbox).await.unwrap(); + let (tx, _rx) = mpsc::channel(1); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + let _ = state.supervisor_sessions.register( + sandbox.object_id().to_string(), + "retained-terminal-session".to_string(), + tx, + shutdown_tx, + ); // --- handle_watch_sandbox --- let err = handle_watch_sandbox( diff --git a/crates/openshell-supervisor-network/src/l7/tls.rs b/crates/openshell-supervisor-network/src/l7/tls.rs index d3def44743..4b8be4736e 100644 --- a/crates/openshell-supervisor-network/src/l7/tls.rs +++ b/crates/openshell-supervisor-network/src/l7/tls.rs @@ -8,7 +8,7 @@ //! store, terminates TLS from the client (presenting dynamic certs per hostname), //! inspects the plaintext HTTP, then re-encrypts to upstream using real root CAs. -use miette::{IntoDiagnostic, Result, miette}; +use miette::{IntoDiagnostic, Result, WrapErr, miette}; use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName}; use rustls::{ClientConfig, ServerConfig}; @@ -66,6 +66,76 @@ impl SandboxCa { pub fn cert_pem(&self) -> &str { &self.ca_cert_pem } + + /// Returns the CA private key in PKCS#8 PEM format. + pub fn private_key_pem(&self) -> String { + self.ca_key.serialize_pem() + } + + /// Load a durable CA certificate and matching private key from absolute paths. + pub fn load_from_paths(certificate_path: &Path, private_key_path: &Path) -> Result { + if !certificate_path.is_absolute() || !private_key_path.is_absolute() { + return Err(miette!( + "proxy CA certificate and key paths must be absolute" + )); + } + if certificate_path == private_key_path { + return Err(miette!( + "proxy CA certificate and private key must use different paths" + )); + } + let certificate_pem = std::fs::read_to_string(certificate_path) + .into_diagnostic() + .wrap_err_with(|| { + format!("read proxy CA certificate {}", certificate_path.display()) + })?; + let private_key_pem = std::fs::read_to_string(private_key_path) + .into_diagnostic() + .wrap_err_with(|| { + format!("read proxy CA private key {}", private_key_path.display()) + })?; + Self::from_pem(&certificate_pem, &private_key_pem) + } + + /// Load a durable CA while preserving the exact certificate bytes supplied + /// by the provisioner for boundary launch replay. + pub fn from_pem(certificate_pem: &str, private_key_pem: &str) -> Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let ca_key = KeyPair::from_pem(private_key_pem) + .into_diagnostic() + .wrap_err("parse proxy CA private key")?; + let certificates = rustls_pemfile::certs(&mut certificate_pem.as_bytes()) + .collect::, _>>() + .into_diagnostic() + .wrap_err("parse proxy CA certificate")?; + if certificates.len() != 1 { + return Err(miette!( + "proxy CA certificate file must contain exactly one certificate" + )); + } + let private_key = rustls_pemfile::private_key(&mut private_key_pem.as_bytes()) + .into_diagnostic() + .wrap_err("parse proxy CA private key")? + .ok_or_else(|| miette!("proxy CA private key file contains no private key"))?; + ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, private_key) + .into_diagnostic() + .wrap_err("proxy CA certificate and private key do not match")?; + + let params = CertificateParams::from_ca_cert_pem(certificate_pem) + .into_diagnostic() + .wrap_err("parse proxy CA signing certificate")?; + let ca_cert = params + .self_signed(&ca_key) + .into_diagnostic() + .wrap_err("initialize proxy CA signer")?; + Ok(Self { + ca_cert, + ca_key, + ca_cert_pem: certificate_pem.to_string(), + }) + } } /// A leaf certificate chain and private key for a specific hostname. @@ -561,4 +631,23 @@ mod tests { "bundle should contain at least one cert", ); } + + #[test] + fn durable_ca_round_trip_preserves_certificate_bytes() { + let generated = SandboxCa::generate().unwrap(); + let certificate = generated.cert_pem().to_string(); + let private_key = generated.private_key_pem(); + let loaded = SandboxCa::from_pem(&certificate, &private_key).unwrap(); + + assert_eq!(loaded.cert_pem(), certificate); + assert_eq!(loaded.private_key_pem(), private_key); + } + + #[test] + fn durable_ca_rejects_mismatched_key_and_relative_paths() { + let certificate = SandboxCa::generate().unwrap(); + let other_key = SandboxCa::generate().unwrap(); + assert!(SandboxCa::from_pem(certificate.cert_pem(), &other_key.private_key_pem()).is_err()); + assert!(SandboxCa::load_from_paths(Path::new("ca.pem"), Path::new("ca.key")).is_err()); + } } diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index a828f75fba..f5537d28f3 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -19,6 +19,7 @@ pub mod procfs; pub mod proxy; pub mod run; pub mod sigv4; +mod spiffe_endpoint; mod token_grant; pub mod upstream_proxy; diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 63aa2c2c70..b9d9316d2e 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -810,6 +810,7 @@ impl OpaEngine { /// generation comparison and callback linearizes state derived from an OPA /// snapshot with every policy reload and fail-closed transition. Callers /// must not perform I/O or other long-running work in `operation`. + #[allow(dead_code)] pub(crate) fn with_current_generation( &self, expected_generation: u64, diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs index ad6095efa9..15ef423d83 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -9,6 +9,7 @@ use super::{PolicyDnsService, SocketTrustedResolver, wire}; use crate::opa::OpaEngine; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::net::set_tcp_nodelay_best_effort; +use openshell_isolation_interface::contract::{DnsMediationSource, DnsTransport}; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; @@ -63,6 +64,75 @@ pub(crate) struct PolicyDnsRuntime { } impl PolicyDnsRuntime { + /// Start policy DNS over an isolation-backend exchange source. No UDP or + /// TCP listener is bound in the supervisor namespace. + pub(crate) fn start_mediated( + policy: Arc, + source: Arc, + trusted_host_gateway: Option, + config: PolicyDnsRuntimeConfig, + mut engine_ready: tokio::sync::watch::Receiver, + ) -> Result { + let upstream = trusted_resolver_from_resolv_conf()?; + let store = Arc::new(ResolvedEndpointStore::new( + StoreConfig::new(config.pools, MAX_MAPPINGS) + .map_err(|error| miette::miette!(error.to_string()))?, + )); + let service = Arc::new(PolicyDnsService::new( + policy, + SocketTrustedResolver::new(upstream), + store.clone(), + trusted_host_gateway, + )); + let task = tokio::spawn(async move { + if engine_ready.wait_for(|ready| *ready).await.is_err() { + return; + } + loop { + let Ok(query) = source.accept().await else { + return; + }; + let service = service.clone(); + tokio::spawn(async move { + let response = match query.transport { + DnsTransport::Udp => { + wire::handle_udp_query_with_ipv6(&service, &query.request, false).await + } + DnsTransport::Tcp => { + wire::handle_tcp_query_with_ipv6(&service, &query.request, false).await + } + } + .map_err(|error| { + openshell_isolation_interface::contract::BackendError::Process(format!( + "policy DNS response failed: {error}" + )) + }); + let _ = query.response.send(response); + }); + } + }); + let expiry_store = store.clone(); + let expiry_task = tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + let _ = expiry_store.expire(std::time::Instant::now()); + } + }); + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "ready") + .message("Policy DNS connected to isolation boundary") + .build() + ); + Ok(Self { + store, + tasks: vec![task, expiry_task], + }) + } + pub(crate) fn start( policy: Arc, udp: tokio::net::UdpSocket, diff --git a/crates/openshell-supervisor-network/src/policy_dns/store.rs b/crates/openshell-supervisor-network/src/policy_dns/store.rs index ea2cd7c0f8..439cdcfa36 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/store.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/store.rs @@ -73,6 +73,17 @@ pub(crate) struct MappingLookup { } impl MappingLookup { + pub(crate) fn pinned_addresses(&self) -> Vec { + let mut seen = HashSet::new(); + self.record + .contracts + .iter() + .filter(|contract| contract.port == self.port) + .flat_map(|contract| contract.pinned_addresses.iter().copied()) + .filter(|address| seen.insert(*address)) + .collect() + } + pub(crate) fn endpoint_ids(&self) -> impl Iterator { self.record .contracts diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 51dd0007af..0ffb64aeda 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -10,8 +10,9 @@ mod relay; use crate::identity::BinaryIdentityCache; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; +use crate::policy_dns::ResolvedEndpointStore; #[cfg(target_os = "linux")] -use crate::policy_dns::{MappingLookupError, PolicyEndpointId, ResolvedEndpointStore}; +use crate::policy_dns::{MappingLookupError, PolicyEndpointId}; use crate::policy_local::{POLICY_LOCAL_HOST, PolicyLocalContext}; use crate::upstream_proxy::{self, UpstreamProxyConfig}; use miette::{IntoDiagnostic, Result}; @@ -24,6 +25,10 @@ use openshell_core::net::{ use openshell_core::policy::ProxyPolicy; use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; use openshell_core::secrets::{self, SecretResolver, rewrite_header_line_checked}; +use openshell_isolation_interface::contract::{ + BinaryIdentity as ContractBinaryIdentity, BoundaryDuplexStream, NetworkMediationSource, + ResolveError, +}; use openshell_ocsf::{ ActionId, ActivityId, AiModel, ApiActivityBuilder, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, Process, SeverityId, StatusId, @@ -36,16 +41,26 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; use tokio::io::{ - AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, AsyncWriteExt, + AsyncBufReadExt, AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, + AsyncWriteExt, }; -use tokio::net::{TcpListener, TcpStream}; +use tokio::net::TcpListener; +#[cfg(any(target_os = "linux", test))] +use tokio::net::TcpStream; use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{debug, warn}; +type ProxyClient = tokio::io::BufReader; + +enum ProxyAcceptError { + Listener(std::io::Error), + Source(openshell_isolation_interface::contract::BackendError), +} + use self::destination::{ - DestinationDenial, DestinationDenialKind, DestinationRequest, build_validation_plan, - validate_destination, + DestinationDenial, DestinationDenialKind, DestinationRequest, build_pinned_validation_plan, + build_validation_plan, validate_destination, }; use self::egress::{ EgressDecision, EgressIntent, EndpointDecision, IdentityUnavailableReason, L7ConfigSnapshot, @@ -256,6 +271,8 @@ impl ProxyHandle { engine_ready: tokio::sync::watch::Receiver, upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, backend_host_gateway: Option, + network_mediation_source: Option>, + policy_dns_store: Option>, ) -> Result { // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no @@ -271,15 +288,27 @@ impl ProxyHandle { )); } - let listener = TcpListener::bind(http_addr).await.into_diagnostic()?; - let local_addr = listener.local_addr().into_diagnostic()?; + let source_backed = network_mediation_source.is_some(); + let listener = if source_backed { + None + } else { + Some(TcpListener::bind(http_addr).await.into_diagnostic()?) + }; + let local_addr = match listener.as_ref() { + Some(listener) => listener.local_addr().into_diagnostic()?, + None => http_addr, + }; { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Listen) .severity(SeverityId::Informational) .status(StatusId::Success) .dst_endpoint(Endpoint::from_ip(local_addr.ip(), local_addr.port())) - .message(format!("Proxy listening on {local_addr}")) + .message(if source_backed { + "Proxy consuming isolation-boundary streams".to_string() + } else { + format!("Proxy listening on {local_addr}") + }) .build(); ocsf_emit!(event); } @@ -373,11 +402,39 @@ impl ProxyHandle { let mut consecutive_resource_errors: u32 = 0; let mut consecutive_unknown_errors: u32 = 0; loop { - match listener.accept().await { - Ok((stream, _addr)) => { + let accepted = if let Some(source) = network_mediation_source.as_ref() { + source + .accept() + .await + .map(|connection| { + ( + connection.stream, + Some(connection.binary_identity), + None, + connection.destination, + ) + }) + .map_err(ProxyAcceptError::Source) + } else { + let listener = listener + .as_ref() + .expect("listener exists without a mediation source"); + listener + .accept() + .await + .map(|(stream, _)| { + set_tcp_nodelay_best_effort(&stream); + let workload_addr = stream.peer_addr().ok(); + let proxy_addr = stream.local_addr().ok(); + let stream: BoundaryDuplexStream = Box::new(stream); + (stream, None, workload_addr.zip(proxy_addr), None) + }) + .map_err(ProxyAcceptError::Listener) + }; + match accepted { + Ok((stream, supplied_identity, socket_addrs, transparent_destination)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; - set_tcp_nodelay_best_effort(&stream); let opa = opa_engine.clone(); let cache = identity_cache.clone(); let spid = entrypoint_pid.clone(); @@ -389,6 +446,7 @@ impl ProxyHandle { let backend_gw = backend_host_gateway.clone(); let up_proxy = upstream_proxy.clone(); let credentials = provider_credentials.clone(); + let dns_store = policy_dns_store.clone(); let resolver = provider_credentials .as_ref() .and_then(ProviderCredentialState::resolver); @@ -401,8 +459,12 @@ impl ProxyHandle { let atx = activity_tx.clone(); tokio::spawn(async move { #[allow(clippy::large_futures)] - if let Err(err) = handle_tcp_connection( - stream, + if let Err(err) = handle_mediated_connection( + tokio::io::BufReader::new(stream), + supplied_identity, + socket_addrs, + transparent_destination, + dns_store, opa, cache, spid, @@ -431,7 +493,19 @@ impl ProxyHandle { } }); } - Err(err) => { + Err(ProxyAcceptError::Source(err)) => { + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message(format!( + "Network-mediation source failed; proxy accept loop exiting: {err}" + )) + .build(); + ocsf_emit!(event); + break; + } + Err(ProxyAcceptError::Listener(err)) => { match classify_accept_error( &err, &mut consecutive_resource_errors, @@ -469,7 +543,7 @@ impl ProxyHandle { }); Ok(Self { - http_addr: Some(local_addr), + http_addr: (!source_backed).then_some(local_addr), join, exited_rx: Some(exited_rx), }) @@ -1196,21 +1270,24 @@ fn middleware_uninspectable_gate( Ok(crate::l7::middleware::uninspectable_traffic_gate(&chain)) } -async fn peek_tunnel_protocol(client: &TcpStream) -> Result> { - let mut peek_buf = [0u8; TUNNEL_PROTOCOL_PEEK_BYTES]; +async fn peek_tunnel_protocol(client: &mut C) -> Result> +where + C: tokio::io::AsyncBufRead + Unpin, +{ let deadline = tokio::time::Instant::now() + TUNNEL_PROTOCOL_PEEK_TIMEOUT; loop { - let n = client.peek(&mut peek_buf).await.into_diagnostic()?; - if n == 0 { + let available = client.fill_buf().await.into_diagnostic()?; + if available.is_empty() { return Ok(None); } - let peek = &peek_buf[..n]; + let n = available.len().min(TUNNEL_PROTOCOL_PEEK_BYTES); + let peek = &available[..n]; let protocol = classify_tunnel_protocol(peek); if protocol != TunnelProtocol::Unsupported || !could_be_supported_tunnel_protocol_prefix(peek) - || n == peek_buf.len() + || n == TUNNEL_PROTOCOL_PEEK_BYTES || tokio::time::Instant::now() >= deadline { return Ok(Some(protocol)); @@ -1588,8 +1665,8 @@ fn build_forward_destination_deny_ocsf_event( } #[allow(clippy::too_many_arguments)] -async fn deny_connect_destination( - client: &mut TcpStream, +async fn deny_connect_destination( + client: &mut C, denial: &DestinationDenial, peer_addr: SocketAddr, host: &str, @@ -1601,7 +1678,10 @@ async fn deny_connect_destination( decision: &EgressDecision, denial_tx: &Option>, activity_tx: &Option, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ let detail = destination_denial_detail(denial.kind); ocsf_emit!(build_connect_destination_deny_ocsf_event( denial, peer_addr, host, port, binary, pid, ancestors, cmdline, @@ -1634,8 +1714,8 @@ async fn deny_connect_destination( } #[allow(clippy::too_many_arguments)] -async fn deny_forward_destination( - client: &mut TcpStream, +async fn deny_forward_destination( + client: &mut C, denial: &DestinationDenial, peer_addr: SocketAddr, method: &str, @@ -1650,7 +1730,10 @@ async fn deny_forward_destination( decision: &EgressDecision, denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ let detail = destination_denial_detail(denial.kind); ocsf_emit!(build_forward_destination_deny_ocsf_event( denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, @@ -1685,9 +1768,105 @@ async fn deny_forward_destination( // Many distinct, non-related context parameters are required for a CONNECT // dispatch; bundling them into a struct would just shift the noise into call // sites. +#[cfg(test)] #[allow(clippy::too_many_arguments)] async fn handle_tcp_connection( - mut client: TcpStream, + client: TcpStream, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + tls_state: Option>, + inference_ctx: Option>, + policy_local_ctx: Option>, + agent_proposals: openshell_core::proposals::AgentProposals, + backend_host_gateway: Arc>, + trusted_host_gateway: Arc>, + upstream_proxy: Arc>, + provider_credentials: Option, + secret_resolver: Option>, + dynamic_credentials: Option< + Arc< + std::sync::RwLock< + std::collections::HashMap, + >, + >, + >, + denial_tx: Option>, + activity_tx: Option, +) -> Result<()> { + let socket_addrs = client.peer_addr().ok().zip(client.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(client); + Box::pin(handle_mediated_connection( + tokio::io::BufReader::new(stream), + None, + socket_addrs, + None, + None, + opa_engine, + identity_cache, + entrypoint_pid, + tls_state, + inference_ctx, + policy_local_ctx, + agent_proposals, + backend_host_gateway, + trusted_host_gateway, + upstream_proxy, + provider_credentials, + secret_resolver, + dynamic_credentials, + denial_tx, + activity_tx, + )) + .await +} + +/// Adapt a transparent application stream to the existing CONNECT pipeline. +/// The synthetic CONNECT request is supervisor-owned and its successful 200 +/// response is consumed before bytes are returned to the workload. +fn virtual_connect_stream( + workload: BoundaryDuplexStream, + authority: String, +) -> BoundaryDuplexStream { + let (handler, bridge) = tokio::io::duplex(64 * 1024); + let (mut bridge_read, mut bridge_write) = tokio::io::split(bridge); + let (mut workload_read, mut workload_write) = tokio::io::split(workload); + tokio::spawn(async move { + let request = format!("CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n"); + if bridge_write.write_all(request.as_bytes()).await.is_ok() { + let _ = tokio::io::copy(&mut workload_read, &mut bridge_write).await; + } + let _ = bridge_write.shutdown().await; + }); + tokio::spawn(async move { + let mut header = Vec::with_capacity(256); + let mut byte = [0_u8; 1]; + while header.len() < MAX_HEADER_BYTES { + match bridge_read.read(&mut byte).await { + Ok(0) | Err(_) => return, + Ok(_) => header.push(byte[0]), + } + if header.ends_with(b"\r\n\r\n") { + break; + } + } + if !header.starts_with(b"HTTP/1.1 200 ") && !header.starts_with(b"HTTP/1.0 200 ") { + let _ = workload_write.shutdown().await; + return; + } + let _ = tokio::io::copy(&mut bridge_read, &mut workload_write).await; + let _ = workload_write.shutdown().await; + }); + Box::new(handler) +} + +#[allow(clippy::too_many_arguments)] +async fn handle_mediated_connection( + mut client: ProxyClient, + supplied_identity: Option>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, + transparent_destination: Option, + policy_dns_store: Option>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -1710,6 +1889,28 @@ async fn handle_tcp_connection( denial_tx: Option>, activity_tx: Option, ) -> Result<()> { + let transparent_mapping = if let Some(destination) = transparent_destination { + let store = policy_dns_store + .as_ref() + .ok_or_else(|| miette::miette!("transparent connection arrived without policy DNS"))?; + let mapping = store + .lookup( + destination.ip(), + destination.port(), + opa_engine.current_generation(), + std::time::Instant::now(), + ) + .map_err(|error| miette::miette!("transparent destination denied: {error}"))?; + let authority = format!( + "{}:{}", + mapping.record.normalized_name.as_str(), + destination.port() + ); + client = tokio::io::BufReader::new(virtual_connect_stream(client.into_inner(), authority)); + Some(mapping) + } else { + None + }; let mut buf = vec![0u8; MAX_HEADER_BYTES]; let mut used = 0usize; @@ -1762,6 +1963,8 @@ async fn handle_tcp_connection( &buf[..], used, &mut client, + supplied_identity.as_ref(), + socket_addrs, opa_engine, identity_cache, entrypoint_pid, @@ -1809,22 +2012,31 @@ async fn handle_tcp_connection( return Ok(()); } - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); // Evaluate OPA policy with process-identity binding. // Wrapped in spawn_blocking because identity resolution does heavy sync I/O: // /proc scanning + SHA256 hashing of binaries (e.g. node at 124MB). - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); let intent = EgressIntent::connect(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity.as_ref() { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -1962,6 +2174,13 @@ async fn handle_tcp_connection( return Ok(()); } } + if let Some(mapping) = transparent_mapping.as_ref() { + decision.endpoint.destination = Some( + build_pinned_validation_plan(mapping.pinned_addresses()).map_err(|denial| { + miette::miette!("transparent destination mapping denied: {}", denial.reason) + })?, + ); + } let destination_plan = decision .endpoint .destination @@ -2211,7 +2430,7 @@ async fn handle_tcp_connection( // Auto-detect the tunnel payload. L7-configured endpoints must only // enter relays that can enforce their configured protocol; unsupported // bytes fail closed below instead of falling through to raw relay. - let Some(tunnel_protocol) = peek_tunnel_protocol(&client).await? else { + let Some(tunnel_protocol) = peek_tunnel_protocol(&mut client).await? else { return Ok(()); }; @@ -2786,6 +3005,77 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres } } +/// Evaluate an egress intent using identity already bound to the accepted +/// connection by an isolation backend. This is the RFC 0012 path; legacy +/// listeners continue to resolve through procfs in `authorize_egress_intent`. +fn authorize_supplied_identity( + engine: &OpaEngine, + intent: EgressIntent, + identity: &Result, +) -> EgressDecision { + let deny = |reason: String, + binary: Option, + ancestors: Vec, + cmdline_paths: Vec| EgressDecision { + intent: intent.clone(), + action: NetworkAction::Deny { reason }, + policy_generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), + endpoint: EndpointDecision::default(), + binary, + binary_pid: None, + ancestors, + cmdline_paths, + }; + + let identity = match identity { + Ok(identity) => identity, + Err(error) => { + return deny( + format!("backend identity resolution failed: {error}"), + None, + vec![], + vec![], + ); + } + }; + let Some(digest) = identity.binary_digest else { + return deny( + "backend identity did not include the required binary digest".to_string(), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ); + }; + let input = crate::opa::NetworkInput { + host: intent.destination.host.clone(), + port: intent.destination.port, + binary_path: identity.binary_path.clone(), + binary_sha256: digest.to_string(), + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }; + match engine.authorize_egress(&input) { + Ok(authorization) => EgressDecision { + intent, + action: authorization.action.clone(), + policy_generation: authorization.generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::from_authorization(&authorization), + binary: Some(identity.binary_path.clone()), + binary_pid: None, + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }, + Err(error) => deny( + format!("policy evaluation error: {error}"), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ), + } +} + /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] fn authorize_egress_intent( @@ -2830,13 +3120,16 @@ const INITIAL_INFERENCE_BUF: usize = 65536; /// /// Returns [`InferenceOutcome::Routed`] if at least one request was successfully /// routed, or [`InferenceOutcome::Denied`] with a reason for all denial cases. -async fn handle_inference_interception( - client: TcpStream, +async fn handle_inference_interception( + client: S, host: &str, port: u16, tls_state: Option<&Arc>, inference_ctx: Option<&Arc>, -) -> Result { +) -> Result +where + S: TokioAsyncRead + TokioAsyncWrite + Unpin + Send, +{ let Some(ctx) = inference_ctx else { return Ok(InferenceOutcome::Denied { reason: "cluster inference context not configured".to_string(), @@ -3425,13 +3718,16 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette ocsf_emit!(event); } -async fn reject_stale_connect_policy( - client: &mut TcpStream, +async fn reject_stale_connect_policy( + client: &mut C, host: &str, port: u16, activity_tx: Option<&ActivitySender>, error: miette::Report, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ warn!( host, port, @@ -4826,7 +5122,9 @@ async fn handle_forward_proxy( target_uri: &str, buf: &[u8], used: usize, - client: &mut TcpStream, + client: &mut ProxyClient, + supplied_identity: Option<&Result>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -4931,19 +5229,27 @@ async fn handle_forward_proxy( canonicalize_forward_host_header(&buf[..used], &canonical_authority)?; // 2. Evaluate OPA policy (same identity binding as CONNECT) - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); - - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); let intent = EgressIntent::forward_http(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -5893,6 +6199,17 @@ async fn handle_forward_proxy( ), ) .await?; + client.shutdown().await.into_diagnostic()?; + let mut discard = [0_u8; 1024]; + let _ = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + match client.read(&mut discard).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + }) + .await; } else { respond( client, @@ -6169,8 +6486,9 @@ fn normalize_host(raw_host: &str) -> &str { raw_host.strip_suffix('.').unwrap_or(raw_host) } -async fn respond(client: &mut TcpStream, bytes: &[u8]) -> Result<()> { +async fn respond(client: &mut (impl TokioAsyncWrite + Unpin), bytes: &[u8]) -> Result<()> { client.write_all(bytes).await.into_diagnostic()?; + client.flush().await.into_diagnostic()?; Ok(()) } @@ -6309,11 +6627,14 @@ const TLS_TERMINATION_UNAVAILABLE_DETAIL: &str = "TLS termination unavailable (C /// HTTP status (the flaw this replaces). Returns `true` when the connection was /// refused (the caller must stop) and `false` when the caller should proceed to /// establish the tunnel. -async fn refuse_connect_when_tls_unavailable( - client: &mut TcpStream, +async fn refuse_connect_when_tls_unavailable( + client: &mut C, tls_state_present: bool, effective_tls_skip: bool, -) -> Result { +) -> Result +where + C: TokioAsyncWrite + Unpin, +{ if tls_state_present || effective_tls_skip { return Ok(false); } @@ -6364,6 +6685,98 @@ mod tests { use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; + #[test] + fn supplied_identity_preserves_authorized_endpoint_metadata() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + inspected: + name: inspected + endpoints: + - host: api.example.com + port: 443 + protocol: rest + enforcement: enforce + request_body_credential_rewrite: true + allowed_ips: ["192.0.2.0/24"] + rules: + - allow: { method: GET, path: /allowed } + binaries: + - path: /usr/bin/python3 +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .expect("load policy"); + let identity = Ok(ContractBinaryIdentity { + binary_path: PathBuf::from("/usr/bin/python3"), + binary_digest: Some("00".repeat(32).parse().expect("digest")), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }); + + let mut decision = authorize_supplied_identity( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + &identity, + ); + + assert_eq!(query_allowed_ips(&decision), ["192.0.2.0/24"]); + hydrate_l7_route(&mut decision); + let route = decision + .endpoint + .l7_route + .expect("supplied identity must retain L7 metadata"); + assert_eq!(route.configs.len(), 1); + assert!(route.configs[0].config.request_body_credential_rewrite); + } + + struct FailedMediationSource; + + #[tokio::test] + async fn virtual_connect_is_portless_and_hides_the_synthetic_handshake() { + let (workload, mut workload_peer) = tokio::io::duplex(1024); + let mut handler = virtual_connect_stream(Box::new(workload), "api.example.com:443".into()); + + workload_peer.write_all(b"client-tls").await.unwrap(); + let mut request = vec![0_u8; 128]; + let length = handler.read(&mut request).await.unwrap(); + let request = &request[..length]; + assert!(request.starts_with(b"CONNECT api.example.com:443 HTTP/1.1\r\n")); + + handler + .write_all(b"HTTP/1.1 200 Connection Established\r\n\r\nserver-tls") + .await + .unwrap(); + let mut response = [0_u8; 10]; + workload_peer.read_exact(&mut response).await.unwrap(); + assert_eq!(&response, b"server-tls"); + } + + #[async_trait::async_trait] + impl NetworkMediationSource for FailedMediationSource { + async fn accept( + &self, + ) -> std::result::Result< + openshell_isolation_interface::contract::MediatedConnection, + openshell_isolation_interface::contract::BackendError, + > { + Err( + openshell_isolation_interface::contract::BackendError::Unavailable( + "test source unavailable".to_string(), + ), + ) + } + } + struct DenyWebSocketPreflight; #[tonic::async_trait] @@ -6551,6 +6964,49 @@ network_policies: {} client.await.unwrap() } + #[tokio::test] + async fn terminal_mediation_source_failure_stops_proxy() { + let policy = include_str!("../data/sandbox-policy.rego"); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + policy, + "network_policies: {}", + true, + ) + .expect("engine"), + ); + let (_ready_tx, ready_rx) = tokio::sync::watch::channel(true); + let mut handle = ProxyHandle::start_with_bind_addr( + &ProxyPolicy { http_addr: None }, + Some(([127, 0, 0, 1], 3128).into()), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(1)), + None, + None, + None, + None, + None, + None, + ready_rx, + &upstream_proxy::UpstreamProxyArgs::default(), + None, + Some(Arc::new(FailedMediationSource)), + None, + ) + .await + .expect("proxy starts before source accept"); + let exited = handle + .take_exit_receiver() + .expect("proxy exposes its exit receiver"); + + tokio::time::timeout(std::time::Duration::from_secs(1), exited) + .await + .expect("source failure must stop the proxy") + .expect_err("proxy task drops the exit sender"); + assert!(handle.join.is_finished()); + } + #[tokio::test] async fn malformed_forward_headers_are_rejected_before_route_or_middleware_dispatch() { for host in ["api.example.com", "unmatched.example.com"] { @@ -6636,7 +7092,13 @@ network_policies: .expect("read proxy response"); response }); - let (mut proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let (proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let socket_addrs = proxy_connection + .peer_addr() + .ok() + .zip(proxy_connection.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(proxy_connection); + let mut proxy_connection = tokio::io::BufReader::new(stream); tokio::time::timeout( std::time::Duration::from_secs(30), @@ -6646,6 +7108,8 @@ network_policies: request.as_bytes(), request.len(), &mut proxy_connection, + None, + socket_addrs, engine, Arc::new(BinaryIdentityCache::new()), Arc::new(AtomicU32::new(std::process::id())), @@ -6771,7 +7235,13 @@ network_policies: .await .unwrap(); }); - let (mut proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let (proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let socket_addrs = proxy_connection + .peer_addr() + .ok() + .zip(proxy_connection.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(proxy_connection); + let mut proxy_connection = tokio::io::BufReader::new(stream); let handler = tokio::spawn(async move { handle_forward_proxy( @@ -6780,6 +7250,8 @@ network_policies: request.as_bytes(), request.len(), &mut proxy_connection, + None, + socket_addrs, engine, Arc::new(BinaryIdentityCache::new()), Arc::new(AtomicU32::new(std::process::id())), @@ -7365,14 +7837,14 @@ network_policies: let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let mut client = TcpStream::connect(addr).await.unwrap(); - let (server, _) = listener.accept().await.unwrap(); + let (mut server, _) = listener.accept().await.unwrap(); client .write_all(crate::l7::rest::HTTP2_PRIOR_KNOWLEDGE_PREFACE) .await .unwrap(); - let protocol = peek_tunnel_protocol(&server) + let protocol = peek_tunnel_protocol(&mut tokio::io::BufReader::new(&mut server)) .await .expect("peek should succeed") .expect("client sent bytes"); diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs index 314596b048..55c4dd9099 100644 --- a/crates/openshell-supervisor-network/src/proxy/egress.rs +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![allow(dead_code)] + //! Transport-neutral egress inputs and authorization results. //! //! Explicit proxy adapters normalize their protocol-specific request into an diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 0f29e331ff..e85ba4021f 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -37,6 +37,7 @@ use crate::l7::tls::{ use crate::opa::OpaEngine; use crate::policy_local::PolicyLocalContext; use crate::proxy::ProxyHandle; +use openshell_isolation_interface::contract::{DnsMediationSource, NetworkMediationSource}; #[cfg(target_os = "linux")] pub struct TransparentRuntimeSetup { @@ -155,6 +156,7 @@ pub struct Networking { /// loop so it can publish updated `SandboxPolicy` snapshots that the /// `policy.local` route handler returns to the workload. pub policy_local_ctx: Arc, + _mediated_policy_dns: Option, #[cfg(target_os = "linux")] _policy_dns: Option, #[cfg(target_os = "linux")] @@ -198,6 +200,8 @@ pub async fn run_networking( upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, host_gateway_ip: Option, #[cfg(target_os = "linux")] transparent_runtime: Option, + network_mediation_source: Option>, + dns_mediation_source: Option>, ) -> Result { // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so @@ -314,10 +318,29 @@ pub async fn run_networking( // the proxy, so it's owned here. let identity_cache = opa_engine.map(|_| Arc::new(BinaryIdentityCache::new())); - // Generate ephemeral CA and TLS state for HTTPS L7 inspection. + // Load a provisioned CA when the boundary lifetime outlives this control + // process; otherwise generate an ephemeral CA. // The CA cert is written to disk so sandbox processes can trust it. let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { - match SandboxCa::generate() { + let configured_ca = match ( + std::env::var_os(openshell_core::sandbox_env::PROXY_CA_CERT), + std::env::var_os(openshell_core::sandbox_env::PROXY_CA_KEY), + ) { + (Some(certificate), Some(private_key)) => Some(SandboxCa::load_from_paths( + std::path::Path::new(&certificate), + std::path::Path::new(&private_key), + )?), + (None, None) => None, + _ => { + return Err(miette::miette!( + "{} and {} must be configured together", + openshell_core::sandbox_env::PROXY_CA_CERT, + openshell_core::sandbox_env::PROXY_CA_KEY, + )); + } + }; + let durable_ca = configured_ca.is_some(); + match configured_ca.map_or_else(SandboxCa::generate, Ok) { Ok(ca) => { let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) .unwrap_or_else(|_| openshell_core::container_paths::TLS_ROOT.to_string()); @@ -357,7 +380,11 @@ pub async fn run_networking( .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Enabled, "enabled") - .message("TLS termination enabled: ephemeral CA generated") + .message(if durable_ca { + "TLS termination enabled: provisioned CA loaded" + } else { + "TLS termination enabled: ephemeral CA generated" + }) .build() ); (Some(state), Some(paths)) @@ -403,6 +430,21 @@ pub async fn run_networking( (None, None) }; + let mediated_policy_dns = if let Some(source) = dns_mediation_source { + let engine = opa_engine + .cloned() + .ok_or_else(|| miette::miette!("Mediated DNS requires an OPA engine"))?; + Some(crate::policy_dns::PolicyDnsRuntime::start_mediated( + engine, + source, + None, + crate::policy_dns::PolicyDnsRuntimeConfig::for_epoch(0)?, + engine_ready_rx.clone(), + )?) + } else { + None + }; + let proxy_handle = if matches!(policy.network.mode, NetworkMode::Proxy) { let proxy_policy = policy.network.proxy.as_ref().ok_or_else(|| { miette::miette!("Network mode is set to proxy but no proxy configuration was provided") @@ -450,6 +492,10 @@ pub async fn run_networking( engine_ready_rx, upstream_proxy_args, host_gateway_ip, + network_mediation_source, + mediated_policy_dns + .as_ref() + .map(|runtime| runtime.store.clone()), ) .await?; Some(proxy_handle) @@ -495,6 +541,7 @@ pub async fn run_networking( proxy: proxy_handle, ca_file_paths, policy_local_ctx, + _mediated_policy_dns: mediated_policy_dns, #[cfg(target_os = "linux")] _policy_dns: policy_dns, #[cfg(target_os = "linux")] diff --git a/crates/openshell-supervisor-network/src/spiffe_endpoint.rs b/crates/openshell-supervisor-network/src/spiffe_endpoint.rs new file mode 100644 index 0000000000..b3b6816f1e --- /dev/null +++ b/crates/openshell-supervisor-network/src/spiffe_endpoint.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::Path; + +/// Convert a path to a SPIFFE Workload API endpoint URL. +/// +/// If the path already has a scheme (`unix:` or `tcp:`), use it as-is. +/// Otherwise, assume it is a Unix socket path and prepend `unix:`. +#[allow(dead_code)] +pub fn workload_api_endpoint(path: &Path) -> String { + let path = path.to_string_lossy(); + if path.starts_with("unix:") || path.starts_with("tcp:") { + path.into_owned() + } else { + format!("unix:{path}") + } +} diff --git a/crates/openshell-supervisor-process/src/boundary_exec.rs b/crates/openshell-supervisor-process/src/boundary_exec.rs index eea12cac38..0ce2e5aa48 100644 --- a/crates/openshell-supervisor-process/src/boundary_exec.rs +++ b/crates/openshell-supervisor-process/src/boundary_exec.rs @@ -106,8 +106,15 @@ impl LocalBoundaryExec { if self.enforcement_mode.enforces_child_sandbox() { crate::sandbox::linux::log_sandbox_readiness(&self.policy, workdir); } - crate::process::prepare_child_sandbox(&self.policy, workdir, self.enforcement_mode) - .map_err(|error| BackendError::Process(error.to_string())) + let runtime_read_only = + crate::process::ca_runtime_read_only_paths(self.ca_file_paths.as_deref()); + crate::process::prepare_child_sandbox( + &self.policy, + workdir, + self.enforcement_mode, + &runtime_read_only, + ) + .map_err(|error| BackendError::Process(error.to_string())) } fn spawn_piped(&self, spec: &ExecSpec) -> Result { @@ -130,7 +137,8 @@ impl LocalBoundaryExec { self.enforcement_mode, #[cfg(target_os = "linux")] prepared, - ); + ) + .map_err(|error| BackendError::Process(error.to_string()))?; #[cfg(target_os = "linux")] let mut child_registry = crate::managed_children::lock(); let mut child = command @@ -232,7 +240,8 @@ impl LocalBoundaryExec { self.enforcement_mode, #[cfg(target_os = "linux")] prepared, - ); + ) + .map_err(|error| BackendError::Process(error.to_string()))?; #[cfg(target_os = "linux")] let mut child_registry = crate::managed_children::lock(); let mut child = command diff --git a/crates/openshell-supervisor-process/src/boundary_io.rs b/crates/openshell-supervisor-process/src/boundary_io.rs index fab37a0062..adc8550d0b 100644 --- a/crates/openshell-supervisor-process/src/boundary_io.rs +++ b/crates/openshell-supervisor-process/src/boundary_io.rs @@ -21,7 +21,7 @@ use openshell_isolation_interface::contract::{ BackendError, BoundaryDuplexStream, BoundaryPortForward, LoopbackTarget, }; use std::collections::HashMap; -use std::os::fd::{AsRawFd, OwnedFd}; +use std::os::fd::OwnedFd; use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Mutex}; @@ -204,13 +204,9 @@ impl BoundaryPortForward for NetnsPortForward { runtime.ensure_active()?; } let addr = std::net::SocketAddr::new(target.host(), target.port()); - let addr_string = addr.to_string(); - let stream = crate::ssh::connect_in_netns( - &addr_string, - self.netns_fd.as_deref().map(AsRawFd::as_raw_fd), - ) - .await - .map_err(|e| BackendError::Process(format!("port-forward connect to {addr}: {e}")))?; + let stream = crate::ssh::connect_in_netns(addr, self.netns_fd.clone()) + .await + .map_err(|e| BackendError::Process(format!("port-forward connect to {addr}: {e}")))?; if let Some(runtime) = &self.runtime { runtime.ensure_active()?; } @@ -314,4 +310,25 @@ mod tests { runtime.unregister_process_group(pid, &second_terminal); assert_eq!(runtime.registered_process_group_count(), 0); } + + #[test] + fn canonical_process_completion_does_not_end_boundary_runtime() { + let runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); + let terminal = Arc::new(std::sync::atomic::AtomicBool::new(true)); + runtime + .register_process_group(42, terminal.clone(), Arc::new(Mutex::new(()))) + .expect("register canonical process"); + + runtime.unregister_process_group(42, &terminal); + + runtime + .ensure_active() + .expect("canonical completion must preserve exec and forwarding"); + assert_eq!(runtime.registered_process_group_count(), 0); + runtime.deactivate(); + assert!(matches!( + runtime.ensure_active(), + Err(BackendError::Terminated(_)) + )); + } } diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs new file mode 100644 index 0000000000..55bdf01e06 --- /dev/null +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -0,0 +1,594 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Additive process and access-plane assembly for a delegated boundary. +//! +//! The established co-located [`crate::run::run_process`] path remains intact. +//! This module exposes the same process primitives to `openshell-sandbox +//! --mode=boundary` and lets `--mode=control` host SSH and gateway relays over +//! placement-neutral boundary interfaces. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::time::Duration; + +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; +use openshell_core::policy::{NetworkMode, SandboxPolicy}; +use openshell_core::proposals::AgentProposals; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward, BoundaryProcess}; +use openshell_ocsf::{ + ActionId, ActivityId, AppLifecycleBuilder, DispositionId, LaunchTypeId, Process as OcsfProcess, + ProcessActivityBuilder, SeverityId, StatusId, ocsf_emit, +}; + +#[cfg(target_os = "linux")] +use crate::netns::NetworkNamespace; +use crate::process::{ + ProcessEnforcementMode, ProcessHandle, ProcessStatus, ResolvedProcessIdentity, + ResolvedWorkspace, +}; + +fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { + openshell_ocsf::ctx::ctx() +} + +/// Host-side SSH and gateway-session tasks for an already-running boundary. +pub struct BoundaryAccess { + instance_id: String, + terminating: Arc, + ssh_task: Option>, + session_task: Option>, + main_session: Option>, +} + +impl BoundaryAccess { + /// Stable supervisor instance ID used for lifecycle reporting. + #[must_use] + pub fn instance_id(&self) -> &str { + &self.instance_id + } + + /// Publish the canonical process's terminal status to attached clients. + pub async fn publish_main_exit(&self, exit_code: i32, attachment_expected: bool) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; + let _ = main_session + .finish_remote(exit_code, attachment_expected) + .await; + } + + /// Release terminal delivery after the gateway acknowledges the exit, then + /// wait for attached clients to consume the terminal status. + pub async fn drain_main_terminal_delivery(&self) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; + main_session.mark_terminal_reported(); + main_session.wait_for_terminal_attachments().await; + } +} + +impl Drop for BoundaryAccess { + fn drop(&mut self) { + self.terminating.store(true, Ordering::Release); + if let Some(task) = self.ssh_task.take() { + task.abort(); + } + if let Some(task) = self.session_task.take() { + task.abort(); + } + } +} + +/// Start the control-owned access plane using boundary-supplied exec and +/// loopback-forwarding capabilities. +#[allow(clippy::too_many_arguments)] +pub async fn start_boundary_access( + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + ssh_socket_path: Option<&str>, + shared_ssh_socket: bool, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + enforcement_mode: ProcessEnforcementMode, + boundary_exec: Arc, + port_forward: Arc, + agent: Arc, +) -> Result { + let instance_id = uuid::Uuid::new_v4().to_string(); + let terminating = Arc::new(AtomicBool::new(false)); + let Some(ssh_socket_path) = ssh_socket_path.map(std::path::PathBuf::from) else { + return Ok(BoundaryAccess { + instance_id, + terminating, + ssh_task: None, + session_task: None, + main_session: None, + }); + }; + + let attachment = agent + .attach() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + let main_session = crate::main_session::MainSession::from_boundary(attachment, agent); + + let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); + let listen_path = ssh_socket_path.clone(); + let ssh_port_forward = port_forward.clone(); + let ssh_main_session = main_session.clone(); + let ssh_task = tokio::spawn(async move { + if let Err(error) = crate::ssh::run_ssh_server( + listen_path, + ssh_ready_tx, + ca_file_paths, + enforcement_mode, + shared_ssh_socket, + ssh_port_forward, + boundary_exec, + Some(ssh_main_session), + ) + .await + { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Critical) + .status(StatusId::Failure) + .message(format!("SSH server failed: {error}")) + .build() + ); + } + }); + + match tokio::time::timeout(Duration::from_secs(10), ssh_ready_rx).await { + Ok(Ok(Ok(()))) => {} + Ok(Ok(Err(error))) => { + ssh_task.abort(); + return Err(error.context("SSH server failed during startup")); + } + Ok(Err(_)) => { + ssh_task.abort(); + return Err(miette::miette!( + "SSH server task ended before signaling readiness" + )); + } + Err(_) => { + ssh_task.abort(); + return Err(miette::miette!( + "SSH server did not start within 10 seconds" + )); + } + } + + let session_task = match (openshell_endpoint, sandbox_id) { + (Some(endpoint), Some(id)) => { + let (task, mut accepted) = crate::supervisor_session::spawn_with_readiness( + endpoint.to_string(), + id.to_string(), + ssh_socket_path, + port_forward, + None, + terminating.clone(), + instance_id.clone(), + ); + match tokio::time::timeout(Duration::from_secs(10), accepted.wait_for(|ready| *ready)) + .await + { + Ok(Ok(_)) => Some(task), + Ok(Err(_)) => { + task.abort(); + return Err(miette::miette!( + "supervisor session ended before gateway acceptance" + )); + } + Err(_) => { + task.abort(); + return Err(miette::miette!( + "gateway did not accept supervisor session within 10 seconds" + )); + } + } + } + _ => None, + }; + + Ok(BoundaryAccess { + instance_id, + terminating, + ssh_task: Some(ssh_task), + session_task, + main_session: Some(main_session), + }) +} + +/// Spawn the admitted workload without placing the gateway or policy authority +/// inside its boundary. +#[allow(clippy::too_many_arguments, clippy::implicit_hasher)] +pub async fn spawn_workload( + program: &str, + args: &[String], + workdir: Option<&str>, + timeout_secs: u64, + interactive: bool, + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + _ssh_socket_path: Option, + _shared_ssh_socket: bool, + policy: &SandboxPolicy, + resolved_process_identity: ResolvedProcessIdentity, + enforcement_mode: ProcessEnforcementMode, + entrypoint_pid: Arc, + entrypoint_started_tx: Option>, + provider_credentials: ProviderCredentialState, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + agent_proposals: AgentProposals, + #[cfg(target_os = "linux")] netns: Option<&NetworkNamespace>, + #[cfg(target_os = "linux")] bypass_denial_tx: Option< + tokio::sync::mpsc::UnboundedSender, + >, + #[cfg(target_os = "linux")] bypass_activity_tx: Option< + openshell_core::activity::ActivitySender, + >, + boundary_runtime: Option>, + instance_id: Option, +) -> Result { + let workspace = ResolvedWorkspace::new(workdir.map(str::to_string), false); + + #[cfg(unix)] + if enforcement_mode.uses_privileged_process_setup() { + crate::process::update_sandbox_passwd_entries() + .wrap_err("update sandbox passwd entries")?; + crate::process::validate_sandbox_user_with_identity(policy, resolved_process_identity) + .wrap_err("validate sandbox process user")?; + crate::process::validate_sandbox_group_with_identity(policy, resolved_process_identity) + .wrap_err("validate sandbox process group")?; + crate::process::prepare_filesystem_with_identity( + policy, + resolved_process_identity, + workspace.root(), + workspace.home().is_some(), + ) + .wrap_err("prepare delegated workload filesystem")?; + } + + crate::run::install_initial_agent_skill(sandbox_id, openshell_endpoint, &agent_proposals).await; + + #[cfg(target_os = "linux")] + crate::process::prepare_supervisor_identity_mount_namespace_from_env() + .wrap_err("prepare supervisor identity mount namespace")?; + crate::sandbox::apply_supervisor_startup_hardening() + .wrap_err("apply supervisor startup hardening")?; + + #[cfg(target_os = "linux")] + let bypass_handle = netns.and_then(|namespace| { + crate::bypass_monitor::spawn( + namespace.name().to_string(), + entrypoint_pid.clone(), + bypass_denial_tx, + bypass_activity_tx, + ) + }); + + #[cfg(target_os = "linux")] + { + let mode = if std::env::var_os("OPENSHELL_REQUIRE_RUNTIME_PID_LIMIT").is_some() { + crate::process::RuntimePidLimitMode::Require + } else { + crate::process::RuntimePidLimitMode::Warn + }; + crate::process::check_runtime_pid_limit(mode).wrap_err("check runtime PID limit")?; + } + + #[cfg(target_os = "linux")] + let boundary_netns_fd = netns + .map(NetworkNamespace::try_clone_ns_fd) + .transpose()? + .flatten() + .map(Arc::new); + #[cfg(not(target_os = "linux"))] + let boundary_netns_fd = None; + + let boundary_runtime = boundary_runtime + .unwrap_or_else(crate::boundary_io::BoundaryRuntimeState::new_exclusive_pid_namespace); + let proxy_url = ssh_proxy_url_for_policy( + policy, + #[cfg(target_os = "linux")] + netns.map(NetworkNamespace::host_ip), + #[cfg(not(target_os = "linux"))] + None, + ); + let user_environment = std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) + .ok() + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_default(); + let port_forward: Arc = + Arc::new(crate::boundary_io::NetnsPortForward::new( + boundary_netns_fd.clone(), + Some(boundary_runtime.clone()), + )); + let boundary_exec: Arc = + Arc::new(crate::boundary_exec::LocalBoundaryExec::new( + policy.clone(), + workspace.owned_root(), + boundary_netns_fd, + proxy_url, + ca_file_paths.clone().map(Arc::new), + provider_credentials, + user_environment, + resolved_process_identity, + enforcement_mode, + boundary_runtime.clone(), + )); + + #[cfg(target_os = "linux")] + let mut handle = ProcessHandle::spawn( + program, + args, + &workspace, + interactive, + policy, + resolved_process_identity, + enforcement_mode, + netns, + ca_file_paths.as_ref(), + &provider_env, + ) + .wrap_err("spawn delegated workload process")?; + #[cfg(not(target_os = "linux"))] + let mut handle = ProcessHandle::spawn( + program, + args, + &workspace, + interactive, + policy, + resolved_process_identity, + enforcement_mode, + ca_file_paths.as_ref(), + &provider_env, + )?; + + entrypoint_pid.store(handle.pid(), Ordering::Release); + if let Some(sender) = entrypoint_started_tx { + let _ = sender.send(handle.pid()); + } + let main_session = crate::main_session::MainSession::new(handle.take_io(), handle.pid()); + let (terminal, signal_lock) = handle.signaling_state(); + boundary_runtime + .register_process_group(handle.pid(), terminal.clone(), signal_lock.clone()) + .map_err(|error| miette::miette!(error.to_string()))?; + + ocsf_emit!( + ProcessActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .launch_type(LaunchTypeId::Spawn) + .process(OcsfProcess::new(program, i64::from(handle.pid()))) + .message(format!("Process started: pid={}", handle.pid())) + .build() + ); + + Ok(SpawnedAgent { + instance_id: instance_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()), + handle, + timeout_secs, + terminal, + signal_lock, + main_session, + #[cfg(target_os = "linux")] + _bypass_handle: bypass_handle, + boundary_exec, + port_forward, + boundary_runtime, + }) +} + +/// Owned workload process and its live boundary capabilities. +pub struct SpawnedAgent { + instance_id: String, + handle: ProcessHandle, + timeout_secs: u64, + terminal: Arc, + signal_lock: Arc>, + main_session: Arc, + #[cfg(target_os = "linux")] + _bypass_handle: Option>, + boundary_exec: Arc, + port_forward: Arc, + boundary_runtime: Arc, +} + +impl SpawnedAgent { + #[must_use] + pub fn instance_id(&self) -> &str { + &self.instance_id + } + + #[must_use] + pub fn pid(&self) -> u32 { + self.handle.pid() + } + + #[must_use] + pub fn signaler(&self) -> AgentSignaler { + AgentSignaler { + pid: self.handle.pid(), + terminal: self.terminal.clone(), + signal_lock: self.signal_lock.clone(), + } + } + + #[must_use] + pub fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } + + #[must_use] + pub fn port_forward(&self) -> Arc { + self.port_forward.clone() + } + + /// Retained canonical-process I/O owned by the boundary. + #[must_use] + pub fn main_session(&self) -> Arc { + self.main_session.clone() + } + + /// Wait for the canonical process to exit, enforcing its admitted + /// wall-clock timeout. Completion does not end the boundary: exec and + /// loopback forwarding remain available until the boundary owner tears + /// down the retained runtime. + pub async fn wait(&mut self) -> Result { + let signaler = self.signaler(); + let status = if self.timeout_secs == 0 { + self.handle.wait().await.into_diagnostic()? + } else if let Ok(status) = + tokio::time::timeout(Duration::from_secs(self.timeout_secs), self.handle.wait()).await + { + status.into_diagnostic()? + } else { + let _ = signaler.term(); + tokio::time::sleep(Duration::from_millis(100)).await; + let _ = signaler.kill(); + self.handle.wait().await.into_diagnostic()? + }; + self.boundary_runtime + .unregister_process_group(self.handle.pid(), &self.terminal); + let _ = self.main_session.finish(status.code(), false).await; + self.main_session.mark_terminal_reported(); + Ok(status) + } +} + +/// Lock-free process-group signal handle used while another task owns `wait`. +#[derive(Clone)] +pub struct AgentSignaler { + pid: u32, + terminal: Arc, + signal_lock: Arc>, +} + +#[cfg(unix)] +impl AgentSignaler { + fn deliver(&self, signal: nix::sys::signal::Signal) -> Result<()> { + let _guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("agent has exited")); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + nix::sys::signal::killpg(nix::unistd::Pid::from_raw(pid), signal).into_diagnostic() + } + + pub fn term(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGTERM) + } + + pub fn kill(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGKILL) + } + + pub fn interrupt(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGINT) + } + + pub fn hangup(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGHUP) + } +} + +fn ssh_proxy_url_for_policy( + policy: &SandboxPolicy, + netns_proxy_host: Option, +) -> Option { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return None; + } + let proxy = policy.network.proxy.as_ref()?; + netns_proxy_host.map_or_else( + || proxy.http_addr.map(|address| format!("http://{address}")), + |host| { + let port = proxy.http_addr.map_or(3128, |address| address.port()); + Some(format!("http://{host}:{port}")) + }, + ) +} + +/// Report the canonical process exit from control mode until acknowledged. +pub async fn report_main_process_exit( + endpoint: &str, + sandbox_id: &str, + instance_id: &str, + exit_code: i32, +) { + let mut delay = Duration::from_millis(250); + loop { + match crate::supervisor_session::report_main_process_exit( + endpoint, + sandbox_id, + instance_id, + exit_code, + ) + .await + { + Ok(()) => break, + Err(error) => { + tracing::warn!(%error, "main-process exit report failed; retrying"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(2)); + } + } + } +} + +/// Finalize canonical process terminal delivery until acknowledged. +pub async fn finalize_main_process_exit(endpoint: &str, sandbox_id: &str, instance_id: &str) { + let mut delay = Duration::from_millis(250); + loop { + match crate::supervisor_session::finalize_main_process_exit( + endpoint, + sandbox_id, + instance_id, + ) + .await + { + Ok(()) => break, + Err(error) => { + tracing::warn!(%error, "main-process finalization failed; retrying"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(2)); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn expected_post_exit_attachment_is_preserved_for_remote_main() { + let main_session = crate::main_session::MainSession::inert(); + let access = BoundaryAccess { + instance_id: "instance".to_string(), + terminating: Arc::new(AtomicBool::new(false)), + ssh_task: None, + session_task: None, + main_session: Some(main_session.clone()), + }; + + access.publish_main_exit(7, true).await; + + main_session + .begin_terminal_attachment() + .expect("declared CLI attachment must remain valid after a fast remote main exits"); + main_session.end_terminal_attachment(); + } +} diff --git a/crates/openshell-supervisor-process/src/lib.rs b/crates/openshell-supervisor-process/src/lib.rs index ee6bedeb22..ba4ce12acd 100644 --- a/crates/openshell-supervisor-process/src/lib.rs +++ b/crates/openshell-supervisor-process/src/lib.rs @@ -12,6 +12,7 @@ pub mod boundary_exec; pub mod boundary_io; pub mod child_env; pub mod debug_rpc; +pub mod delegated; #[cfg(unix)] pub mod identity; pub mod log_push; diff --git a/crates/openshell-supervisor-process/src/main_session.rs b/crates/openshell-supervisor-process/src/main_session.rs index 00dd2ea53c..2aba0c48c1 100644 --- a/crates/openshell-supervisor-process/src/main_session.rs +++ b/crates/openshell-supervisor-process/src/main_session.rs @@ -17,6 +17,10 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::Notify; use tokio::sync::watch; +use openshell_isolation_interface::contract::{ + BoundaryProcess, BoundarySignal, BoundaryTerminal, ProcessAttachment, +}; + use crate::process::ProcessIo; const OUTPUT_BUFFER_BYTES: usize = 1024 * 1024; @@ -186,6 +190,8 @@ pub struct MainSession { input_owner: Mutex>, next_owner: AtomicU64, pty_master: Option>, + boundary_process: Option>, + boundary_terminal: Option>, readers_remaining: AtomicUsize, readers_done: Notify, finished: std::sync::atomic::AtomicBool, @@ -194,6 +200,7 @@ pub struct MainSession { } impl MainSession { + const REMOTE_OUTPUT_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); #[cfg(test)] pub fn inert() -> Arc { let (input, _input_rx) = tokio::sync::mpsc::channel(64); @@ -205,6 +212,8 @@ impl MainSession { input_owner: Mutex::new(None), next_owner: AtomicU64::new(1), pty_master: None, + boundary_process: None, + boundary_terminal: None, readers_remaining: AtomicUsize::new(0), readers_done: Notify::new(), finished: std::sync::atomic::AtomicBool::new(false), @@ -256,6 +265,8 @@ impl MainSession { input_owner: Mutex::new(None), next_owner: AtomicU64::new(1), pty_master, + boundary_process: None, + boundary_terminal: None, readers_remaining: AtomicUsize::new(if terminal { 1 } else { 2 }), readers_done: Notify::new(), finished: std::sync::atomic::AtomicBool::new(false), @@ -270,6 +281,81 @@ impl MainSession { session } + /// Build the control-side multiplexer around a boundary-owned admitted + /// process. Process lifecycle and PTY operations remain delegated to the + /// boundary process handle. + #[must_use] + pub fn from_boundary( + attachment: ProcessAttachment, + process: Arc, + ) -> Arc { + let ProcessAttachment { + stdin, + stdout, + stderr, + terminal, + } = attachment; + let terminal_mode = terminal.is_some(); + let (input, mut input_rx) = tokio::sync::mpsc::channel::>(64); + let session = Arc::new(Self { + pid: 0, + terminal: terminal_mode, + input, + output: OutputLog::new(), + input_owner: Mutex::new(None), + next_owner: AtomicU64::new(1), + pty_master: None, + boundary_process: Some(process), + boundary_terminal: terminal, + readers_remaining: AtomicUsize::new(if terminal_mode { 1 } else { 2 }), + readers_done: Notify::new(), + finished: std::sync::atomic::AtomicBool::new(false), + terminal_attachments: Mutex::new(TerminalAttachmentState { + active: 0, + process_finished: false, + expectation: AttachmentExpectation::None, + }), + terminal_attachments_done: Notify::new(), + }); + let stdout_session = Arc::clone(&session); + tokio::spawn(async move { + let mut stdout = stdout; + let mut buffer = [0u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stdout_session + .publish(MainOutput::Stdout(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stdout_session.reader_finished(); + }); + if let Some(mut stderr) = stderr { + let stderr_session = Arc::clone(&session); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stderr_session + .publish(MainOutput::Stderr(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stderr_session.reader_finished(); + }); + } + tokio::spawn(async move { + let mut stdin = stdin; + while let Some(data) = input_rx.recv().await { + if stdin.write_all(&data).await.is_err() { + break; + } + let _ = stdin.flush().await; + } + }); + session + } + fn start_io( this: &Arc, io: ProcessIo, @@ -380,10 +466,39 @@ impl MainSession { /// /// Returns whether terminal delivery must complete before shutdown. pub async fn finish(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.wait_for_output_readers().await; + self.complete_finish(exit_code, attachment_expected) + } + + /// Finish a remotely owned process without allowing descendants that keep + /// inherited output descriptors open to block terminal publication forever. + pub(crate) async fn finish_remote(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.finish_remote_with_timeout( + exit_code, + attachment_expected, + Self::REMOTE_OUTPUT_DRAIN_TIMEOUT, + ) + .await + } + + async fn finish_remote_with_timeout( + &self, + exit_code: i32, + attachment_expected: bool, + timeout: std::time::Duration, + ) -> bool { + let _ = tokio::time::timeout(timeout, self.wait_for_output_readers()).await; + self.complete_finish(exit_code, attachment_expected) + } + + async fn wait_for_output_readers(&self) { let notified = self.readers_done.notified(); if self.readers_remaining.load(Ordering::Acquire) != 0 { notified.await; } + } + + fn complete_finish(&self, exit_code: i32, attachment_expected: bool) -> bool { let delivery_pending = { let mut state = self .terminal_attachments @@ -499,7 +614,16 @@ impl MainSession { } } - pub fn resize(&self, columns: u32, rows: u32, pixel_width: u32, pixel_height: u32) { + pub async fn resize(&self, columns: u32, rows: u32, pixel_width: u32, pixel_height: u32) { + if let Some(terminal) = self.boundary_terminal.as_ref() { + let _ = terminal + .resize( + u16::try_from(columns.max(1)).unwrap_or(u16::MAX), + u16::try_from(rows.max(1)).unwrap_or(u16::MAX), + ) + .await; + return; + } let Some(master) = self.pty_master.as_ref() else { return; }; @@ -515,9 +639,23 @@ impl MainSession { } } - pub fn signal_group(&self, signal: nix::sys::signal::Signal) -> Result<(), nix::errno::Errno> { + pub async fn signal_group(&self, signal: nix::sys::signal::Signal) -> Result<(), String> { + if let Some(process) = self.boundary_process.as_ref() { + let signal = match signal { + nix::sys::signal::Signal::SIGHUP => BoundarySignal::Hup, + nix::sys::signal::Signal::SIGINT => BoundarySignal::Int, + nix::sys::signal::Signal::SIGKILL => BoundarySignal::Kill, + nix::sys::signal::Signal::SIGTERM => BoundarySignal::Term, + other => return Err(format!("boundary signal {other:?} is unsupported")), + }; + return process + .signal(signal) + .await + .map_err(|error| error.to_string()); + } let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); nix::sys::signal::kill(nix::unistd::Pid::from_raw(-pid), signal) + .map_err(|error| error.to_string()) } #[must_use] @@ -544,6 +682,83 @@ fn set_nonblocking(file: &std::fs::File) -> Result<(), nix::errno::Errno> { #[cfg(test)] mod tests { use super::*; + use openshell_isolation_interface::contract::{ + BackendError, BoundaryExitStatus, BoundaryInput, BoundaryOutput, + }; + + struct TestBoundaryProcess { + signals: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryProcess for TestBoundaryProcess { + async fn wait(&self) -> Result { + Ok(BoundaryExitStatus::Exited(0)) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + self.signals.lock().unwrap().push(signal); + Ok(()) + } + + async fn terminate(&self) -> Result<(), BackendError> { + Ok(()) + } + } + + struct TestBoundaryTerminal { + size: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryTerminal for TestBoundaryTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + *self.size.lock().unwrap() = Some((cols, rows)); + Ok(()) + } + } + + #[tokio::test] + async fn boundary_attachment_drives_main_io_signal_and_terminal() { + let (stdin, mut stdin_peer) = tokio::io::duplex(1024); + let (stdout, mut stdout_peer) = tokio::io::duplex(1024); + let process = Arc::new(TestBoundaryProcess { + signals: Mutex::new(Vec::new()), + }); + let terminal = Arc::new(TestBoundaryTerminal { + size: Mutex::new(None), + }); + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let attachment = ProcessAttachment { + stdin, + stdout, + stderr: None, + terminal: Some(terminal.clone()), + }; + let session = MainSession::from_boundary(attachment, process.clone()); + let mut output = session.subscribe(); + + stdout_peer.write_all(b"ready\n").await.unwrap(); + assert!(matches!( + output.recv().await.unwrap(), + MainOutput::Stdout(data) if data == b"ready\n"[..] + )); + + let (_owner, input) = session.acquire_input().unwrap(); + input.send(b"hello\n".to_vec()).await.unwrap(); + let mut received = [0_u8; 6]; + stdin_peer.read_exact(&mut received).await.unwrap(); + assert_eq!(&received, b"hello\n"); + + session.resize(120, 40, 0, 0).await; + assert_eq!(*terminal.size.lock().unwrap(), Some((120, 40))); + session + .signal_group(nix::sys::signal::Signal::SIGINT) + .await + .unwrap(); + assert_eq!(*process.signals.lock().unwrap(), vec![BoundarySignal::Int]); + } #[test] fn input_lease_has_one_owner_and_can_be_reacquired() { @@ -632,6 +847,27 @@ mod tests { .expect("closing the attachment should wake the waiter"); } + #[tokio::test] + async fn remote_finish_bounds_output_drain_before_publishing_exit() { + let mut session = MainSession::inert(); + Arc::get_mut(&mut session) + .expect("sole test session reference") + .readers_remaining = AtomicUsize::new(1); + let mut output = session.subscribe(); + + session + .finish_remote_with_timeout(19, false, std::time::Duration::from_millis(10)) + .await; + + assert!(matches!( + output + .recv() + .await + .expect("terminal status after bounded drain"), + MainOutput::Exit(19) + )); + } + #[tokio::test] async fn declared_attachment_waits_for_connection_then_natural_close() { let session = MainSession::inert(); diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 2b4ea554ed..186d11e721 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -11,9 +11,12 @@ mod nft_ruleset; use miette::{IntoDiagnostic, Result}; use std::net::IpAddr; +use std::os::fd::AsRawFd as _; +use std::os::fd::{BorrowedFd, OwnedFd}; use std::os::unix::io::RawFd; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Command; +use std::sync::OnceLock; use tracing::{debug, warn}; use uuid::Uuid; @@ -26,13 +29,80 @@ const SANDBOX_IP_SUFFIX: u8 = 2; /// this listener before the bypass fence runs. pub const POLICY_DNS_PORT: u16 = 15_053; pub const TRANSPARENT_TCP_PORT: u16 = 15_001; -const IP_SEARCH_PATHS: &[&str] = &["/usr/sbin/ip", "/sbin/ip", "/usr/bin/ip", "/bin/ip"]; -const NSENTER_SEARCH_PATHS: &[&str] = &[ - "/usr/bin/nsenter", - "/bin/nsenter", - "/usr/sbin/nsenter", - "/sbin/nsenter", -]; +const IP_SEARCH_PATHS: &[&str] = &["usr/sbin/ip", "sbin/ip", "usr/bin/ip", "bin/ip"]; +static TRUSTED_RUNTIME_ROOT: OnceLock = OnceLock::new(); + +/// Pin the driver-owned helper runtime used for conformant namespace setup. +/// +/// VM guest leaves call this before starting any control or workload threads +/// because their executable may be launched through a dynamic loader. In that +/// case `/proc/self/exe` identifies the loader rather than the supervisor +/// binary, so the default executable-relative lookup is not authoritative. +/// +/// # Errors +/// +/// Returns an error for a relative path or if another root was already pinned. +pub fn configure_trusted_runtime_root(root: PathBuf) -> Result<()> { + if !root.is_absolute() { + return Err(miette::miette!( + "trusted supervisor helper runtime root must be absolute" + )); + } + TRUSTED_RUNTIME_ROOT.set(root).map_err(|configured| { + miette::miette!( + "trusted supervisor helper runtime root is already configured as {}", + configured.display() + ) + }) +} + +#[derive(Clone, Debug)] +struct TrustedHelper { + executable: PathBuf, + loader: Option, + library_path: String, + xtables_path: PathBuf, +} + +impl TrustedHelper { + fn command(&self) -> Command { + self.loader.as_ref().map_or_else( + || Command::new(&self.executable), + |loader| { + let mut command = Command::new(loader); + command + .env_clear() + .env("XTABLES_LIBDIR", &self.xtables_path) + .arg("--library-path") + .arg(&self.library_path) + .arg(&self.executable); + command + }, + ) + } + + fn tokio_command(&self) -> tokio::process::Command { + self.loader.as_ref().map_or_else( + || tokio::process::Command::new(&self.executable), + |loader| { + let mut command = tokio::process::Command::new(loader); + command + .env_clear() + .env("XTABLES_LIBDIR", &self.xtables_path) + .arg("--library-path") + .arg(&self.library_path) + .arg(&self.executable); + command + }, + ) + } +} + +#[derive(Clone, Copy, Debug)] +enum HelperSource { + LegacyWorkloadImage, + TrustedSupervisorRuntime, +} /// Handle to a network namespace with veth pair. /// @@ -44,13 +114,24 @@ pub struct NetworkNamespace { /// Host-side veth interface name veth_host: String, /// Sandbox-side veth interface name (inside namespace, used only during setup) - _veth_sandbox: String, + #[allow(dead_code)] + veth_sandbox: String, /// Host-side IP address (proxy binds here) host_ip: IpAddr, /// Sandbox-side IP address sandbox_ip: IpAddr, /// File descriptor for the namespace (for setns) ns_fd: Option, + helper_source: HelperSource, +} + +/// Cloneable coordinates for checking a live ceiling without retaining the +/// namespace fd or delaying namespace cleanup. +#[derive(Clone, Debug)] +pub struct EgressCeilingVerifier { + namespace: String, + host_ip: IpAddr, + helper_source: HelperSource, } impl NetworkNamespace { @@ -66,6 +147,14 @@ impl NetworkNamespace { /// /// Returns an error if namespace creation or network setup fails. pub fn create() -> Result { + Self::create_with_helper_source(HelperSource::LegacyWorkloadImage) + } + + fn create_conformant() -> Result { + Self::create_with_helper_source(HelperSource::TrustedSupervisorRuntime) + } + + fn create_with_helper_source(helper_source: HelperSource) -> Result { let id = Uuid::new_v4(); let short_id = &id.to_string()[..8]; let name = format!("sandbox-{short_id}"); @@ -89,84 +178,101 @@ impl NetworkNamespace { ); // Create the namespace - run_ip(&["netns", "add", &name])?; + run_ip(helper_source, &["netns", "add", &name])?; // Create veth pair - if let Err(e) = run_ip(&[ - "link", - "add", - &veth_host, - "type", - "veth", - "peer", - "name", - &veth_sandbox, - ]) { + if let Err(e) = run_ip( + helper_source, + &[ + "link", + "add", + &veth_host, + "type", + "veth", + "peer", + "name", + &veth_sandbox, + ], + ) { // Cleanup namespace on failure - let _ = run_ip(&["netns", "delete", &name]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Move sandbox veth into namespace - if let Err(e) = run_ip(&["link", "set", &veth_sandbox, "netns", &name]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip( + helper_source, + &["link", "set", &veth_sandbox, "netns", &name], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Configure host side let host_cidr = format!("{host_ip}/24"); - if let Err(e) = run_ip(&["addr", "add", &host_cidr, "dev", &veth_host]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip( + helper_source, + &["addr", "add", &host_cidr, "dev", &veth_host], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } - if let Err(e) = run_ip(&["link", "set", &veth_host, "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip(helper_source, &["link", "set", &veth_host, "up"]) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Configure sandbox side (inside namespace) let sandbox_cidr = format!("{sandbox_ip}/24"); - if let Err(e) = run_ip_netns(&name, &["addr", "add", &sandbox_cidr, "dev", &veth_sandbox]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns( + helper_source, + &name, + &["addr", "add", &sandbox_cidr, "dev", &veth_sandbox], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } - if let Err(e) = run_ip_netns(&name, &["link", "set", &veth_sandbox, "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns(helper_source, &name, &["link", "set", &veth_sandbox, "up"]) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Bring up loopback in namespace - if let Err(e) = run_ip_netns(&name, &["link", "set", "lo", "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns(helper_source, &name, &["link", "set", "lo", "up"]) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Add default route via host let host_ip_str = host_ip.to_string(); - if let Err(e) = run_ip_netns(&name, &["route", "add", "default", "via", &host_ip_str]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns( + helper_source, + &name, + &["route", "add", "default", "via", &host_ip_str], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Open the namespace file descriptor for later use with setns - let ns_path = openshell_core::container_paths::netns_path(&name); + let ns_path = format!("/var/run/netns/{name}"); let ns_fd = match nix::fcntl::open( - ns_path.as_path(), + ns_path.as_str(), nix::fcntl::OFlag::O_RDONLY, nix::sys::stat::Mode::empty(), ) { Ok(fd) => Some(fd), Err(e) => { - warn!(error = %e, "Failed to open namespace fd, will use nsenter fallback"); + warn!(error = %e, "Failed to retain network namespace fd"); None } }; @@ -185,10 +291,11 @@ impl NetworkNamespace { Ok(Self { name, veth_host, - _veth_sandbox: veth_sandbox, + veth_sandbox, host_ip, sandbox_ip, ns_fd, + helper_source, }) } @@ -249,25 +356,21 @@ impl NetworkNamespace { self.ns_fd } - /// Install nftables rules for bypass detection inside the namespace. - /// - /// Sets up OUTPUT chain rules that: - /// 1. ACCEPT traffic destined for the proxy (`host_ip:proxy_port`) - /// 2. ACCEPT loopback traffic - /// 3. ACCEPT established/related connections (response packets) - /// 4. LOG + REJECT all other TCP/UDP traffic (bypass attempts) - /// - /// This provides two benefits: - /// - **Fast-fail UX**: applications get immediate ECONNREFUSED instead of - /// a 30-second timeout when they bypass the proxy - /// - **Diagnostics**: nftables LOG entries are picked up by the bypass - /// monitor to emit structured tracing events - /// - /// Degrades gracefully if `nft` is not available — the namespace - /// still provides isolation via routing, just without fast-fail and - /// diagnostic logging. + /// Duplicate the namespace descriptor for a retained runtime handle. + pub fn try_clone_ns_fd(&self) -> Result> { + self.ns_fd + .map(|fd| { + // SAFETY: `NetworkNamespace` owns `fd` for at least this call. + #[allow(unsafe_code)] + let borrowed = unsafe { BorrowedFd::borrow_raw(fd) }; + borrowed.try_clone_to_owned().into_diagnostic() + }) + .transpose() + } + + /// Install the legacy best-effort nftables bypass-detection rules. pub fn install_bypass_rules(&self, proxy_port: u16) -> Result<()> { - let Some(nft_path) = find_nft() else { + let Some(nft_path) = find_nft(self.helper_source) else { openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Medium) @@ -281,33 +384,25 @@ impl NetworkNamespace { ); return Ok(()); }; - - let host_ip_str = self.host_ip.to_string(); + let host_ip = self.host_ip.to_string(); let log_prefix = format!("openshell:bypass:{}:", &self.name); - - // The kernel's nf_log_syslog module suppresses log output from - // non-init network namespaces by default. Enable it so the bypass - // monitor can see log entries from the sandbox namespace. enable_nf_log_all_netns(); - let commands = - nft_ruleset::generate_bypass_commands(&host_ip_str, proxy_port, Some(&log_prefix)); - - if let Err(e) = run_nft_commands_netns(&self.name, &nft_path, &commands) { + nft_ruleset::generate_bypass_commands(&host_ip, proxy_port, Some(&log_prefix)); + if let Err(error) = run_nft_commands_netns(&self.name, &nft_path, &commands) { openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Medium) .status(openshell_ocsf::StatusId::Failure) .state(openshell_ocsf::StateId::Disabled, "failed") .message(format!( - "Failed to install bypass detection rules [ns:{}]: {e}", + "Failed to install bypass detection rules [ns:{}]: {error}", self.name )) .build() ); - return Err(e); + return Err(error); } - openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Informational) @@ -319,7 +414,6 @@ impl NetworkNamespace { )) .build() ); - Ok(()) } @@ -338,10 +432,11 @@ impl NetworkNamespace { // kernel reaches the nft OUTPUT hook; REDIRECT then reroutes it to // the local transparent listener. run_ip_netns( + self.helper_source, &self.name, &["-6", "route", "replace", synthetic_ipv6_cidr, "dev", "lo"], )?; - let nft_path = find_nft().ok_or_else(|| { + let nft_path = find_nft(self.helper_source).ok_or_else(|| { miette::miette!( "trusted nft helper not found; policy DNS and transparent TCP require nftables" ) @@ -386,8 +481,11 @@ impl NetworkNamespace { .into_diagnostic()?, ]; for family in ["-4", "-6"] { - let routes = - run_ip_netns_output(&self.name, &[family, "route", "show", "table", "all"])?; + let routes = run_ip_netns_output( + self.helper_source, + &self.name, + &[family, "route", "show", "table", "all"], + )?; if let Some((route, pool)) = first_route_overlap(&routes, &reserved) { return Err(miette::miette!( "synthetic address pool {pool} overlaps workload route {route}; refusing to enable policy DNS" @@ -510,6 +608,101 @@ impl NetworkNamespace { )) } + /// Install the RFC 0012 default-deny egress ceiling inside the namespace. + /// + /// Sets up OUTPUT chain rules that: + /// 1. ACCEPT traffic destined for the proxy (`host_ip:proxy_port`) + /// 2. ACCEPT loopback traffic + /// 3. LOG + REJECT TCP/UDP bypass attempts and DROP every other packet + /// + /// This provides two benefits: + /// - **Fast-fail UX**: applications get immediate ECONNREFUSED instead of + /// a 30-second timeout when they bypass the proxy + /// - **Diagnostics**: nftables LOG entries are picked up by the bypass + /// monitor to emit structured tracing events + /// + /// Missing nftables support is fatal: without the default-deny ceiling the + /// backend cannot confirm that all workload egress reaches mediation. + pub fn install_egress_ceiling(&self, proxy_port: u16) -> Result<()> { + let Some(nft_path) = find_nft(self.helper_source) else { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::High) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Disabled, "unavailable") + .message(format!( + "nft not found; refusing to establish the egress ceiling [ns:{}]", + self.name + )) + .build() + ); + return Err(miette::miette!( + "nft not found; cannot establish default-deny egress ceiling" + )); + }; + + let host_ip_str = self.host_ip.to_string(); + let log_prefix = format!("openshell:bypass:{}:", &self.name); + + // The kernel's nf_log_syslog module suppresses log output from + // non-init network namespaces by default. Enable it so the bypass + // monitor can see log entries from the sandbox namespace. + enable_nf_log_all_netns(); + + let commands = nft_ruleset::generate_egress_ceiling_commands( + &host_ip_str, + proxy_port, + Some(&log_prefix), + ); + + if let Err(e) = run_nft_commands_netns(&self.name, &nft_path, &commands) { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::High) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Disabled, "failed") + .message(format!( + "Failed to establish egress ceiling [ns:{}]: {e}", + self.name + )) + .build() + ); + return Err(e); + } + + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "installed") + .message(format!( + "Default-deny egress ceiling established [ns:{}]", + self.name + )) + .build() + ); + + Ok(()) + } + + /// Verify the live default-deny egress ceiling installed for this boundary. + /// + /// This reads the ruleset back from the kernel rather than treating a + /// successful installation attempt as proof that standing enforcement is + /// still present. + pub fn verify_egress_ceiling(&self, proxy_port: u16) -> Result<()> { + self.egress_ceiling_verifier().verify(proxy_port) + } + + #[must_use] + pub fn egress_ceiling_verifier(&self) -> EgressCeilingVerifier { + EgressCeilingVerifier { + namespace: self.name.clone(), + host_ip: self.host_ip, + helper_source: self.helper_source, + } + } + /// Bind a TCP listener inside this network namespace on a dedicated thread. /// /// Spawns a short-lived OS thread that enters the namespace via `setns`, @@ -548,6 +741,158 @@ impl NetworkNamespace { } } +impl EgressCeilingVerifier { + fn nft_helper(&self) -> Result { + find_nft(self.helper_source) + .ok_or_else(|| miette::miette!("nft not found; cannot verify egress ceiling")) + } + + fn verify(&self, proxy_port: u16) -> Result<()> { + let nft = self.nft_helper()?; + let output = trusted_command_in_netns(&nft, &self.namespace)? + .args(["-j", "list", "chain", "inet", "openshell_bypass", "output"]) + .output() + .into_diagnostic()?; + self.validate_output(proxy_port, &output) + } + + /// Run a verifier helper with a hard deadline. Dropping the timed-out + /// future kills the child, so a stuck `nft` cannot retain the + /// namespace or suspend enforcement-loss detection indefinitely. + pub async fn verify_bounded( + &self, + proxy_port: u16, + timeout: std::time::Duration, + ) -> Result<()> { + let nft = self.nft_helper()?; + let mut command = trusted_tokio_command_in_netns(&nft, &self.namespace)?; + command.kill_on_drop(true).args([ + "-j", + "list", + "chain", + "inet", + "openshell_bypass", + "output", + ]); + let output = tokio::time::timeout(timeout, command.output()) + .await + .map_err(|_| miette::miette!("egress ceiling verification timed out"))? + .into_diagnostic()?; + self.validate_output(proxy_port, &output) + } + + fn validate_output(&self, proxy_port: u16, output: &std::process::Output) -> Result<()> { + if !output.status.success() { + return Err(miette::miette!( + "could not read back egress ceiling in netns {}: {}", + self.namespace, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + verify_egress_ceiling_json(&output.stdout, &self.host_ip.to_string(), proxy_port) + } +} + +fn verify_egress_ceiling_json(json: &[u8], host_ip: &str, proxy_port: u16) -> Result<()> { + let document: serde_json::Value = serde_json::from_slice(json).into_diagnostic()?; + let objects = document + .get("nftables") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| miette::miette!("nft response has no nftables object list"))?; + + let chain_is_default_deny = objects.iter().any(|object| { + let Some(chain) = object.get("chain") else { + return false; + }; + chain.get("family").and_then(serde_json::Value::as_str) == Some("inet") + && chain.get("table").and_then(serde_json::Value::as_str) == Some("openshell_bypass") + && chain.get("name").and_then(serde_json::Value::as_str) == Some("output") + && chain.get("type").and_then(serde_json::Value::as_str) == Some("filter") + && chain.get("hook").and_then(serde_json::Value::as_str) == Some("output") + && chain.get("prio").and_then(serde_json::Value::as_i64) == Some(0) + && chain.get("policy").and_then(serde_json::Value::as_str) == Some("drop") + }); + if !chain_is_default_deny { + return Err(miette::miette!( + "egress ceiling output chain is absent or not policy drop" + )); + } + + let output_rules: Vec<&serde_json::Value> = objects + .iter() + .filter_map(|object| object.get("rule")) + .collect(); + for rule in &output_rules { + let expressions = rule + .get("expr") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| miette::miette!("egress ceiling rule has no expression list"))?; + for expression in expressions { + let keys = expression + .as_object() + .ok_or_else(|| miette::miette!("egress ceiling contains a malformed expression"))?; + if keys.len() != 1 + || !keys.keys().all(|key| { + matches!( + key.as_str(), + "match" | "counter" | "limit" | "log" | "reject" | "drop" | "accept" + ) + }) + { + return Err(miette::miette!( + "egress ceiling contains an unsupported or redirecting expression" + )); + } + } + } + let accept_rules: Vec<&serde_json::Value> = output_rules + .into_iter() + .filter(|rule| { + rule.get("family").and_then(serde_json::Value::as_str) == Some("inet") + && rule.get("table").and_then(serde_json::Value::as_str) == Some("openshell_bypass") + && rule.get("chain").and_then(serde_json::Value::as_str) == Some("output") + }) + .filter(|rule| { + rule.get("expr") + .and_then(serde_json::Value::as_array) + .is_some_and(|expressions| { + expressions + .iter() + .any(|expression| expression == &serde_json::json!({"accept": null})) + }) + }) + .collect(); + let proxy_expressions = serde_json::json!([ + {"match":{"op":"==","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":host_ip}}, + {"match":{"op":"==","left":{"payload":{"protocol":"tcp","field":"dport"}},"right":proxy_port}}, + {"accept":null} + ]); + let loopback_expressions = serde_json::json!([ + {"match":{"op":"==","left":{"meta":{"key":"oifname"}},"right":"lo"}}, + {"accept":null} + ]); + let mut proxy_allowed = false; + let mut loopback_allowed = false; + for rule in accept_rules { + let expressions = rule.get("expr").expect("accept rule has expressions"); + if expressions == &proxy_expressions { + proxy_allowed = true; + } else if expressions == &loopback_expressions { + loopback_allowed = true; + } else { + return Err(miette::miette!( + "egress ceiling contains an unexpected accept rule: {expressions}" + )); + } + } + if !proxy_allowed || !loopback_allowed { + return Err(miette::miette!( + "egress ceiling is missing the proxy or loopback accept rule" + )); + } + Ok(()) +} + impl Drop for NetworkNamespace { fn drop(&mut self) { debug!(namespace = %self.name, "Cleaning up network namespace"); @@ -558,7 +903,9 @@ impl Drop for NetworkNamespace { } // Delete the host-side veth (this also removes the peer) - if let Err(e) = run_ip(&["link", "delete", &self.veth_host]) { + let mut cleanup_failed = false; + if let Err(e) = run_ip(self.helper_source, &["link", "delete", &self.veth_host]) { + cleanup_failed = true; warn!( error = %e, veth = %self.veth_host, @@ -567,7 +914,8 @@ impl Drop for NetworkNamespace { } // Delete the namespace - if let Err(e) = run_ip(&["netns", "delete", &self.name]) { + if let Err(e) = run_ip(self.helper_source, &["netns", "delete", &self.name]) { + cleanup_failed = true; warn!( error = %e, namespace = %self.name, @@ -575,14 +923,32 @@ impl Drop for NetworkNamespace { ); } - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Disabled, "cleaned_up") - .message(format!("Network namespace cleaned up [ns:{}]", self.name)) - .build() - ); + let event = openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(if cleanup_failed { + openshell_ocsf::SeverityId::High + } else { + openshell_ocsf::SeverityId::Informational + }) + .status(if cleanup_failed { + openshell_ocsf::StatusId::Failure + } else { + openshell_ocsf::StatusId::Success + }) + .state( + openshell_ocsf::StateId::Disabled, + if cleanup_failed { + "cleanup_failed" + } else { + "cleaned_up" + }, + ) + .message(if cleanup_failed { + format!("Network namespace cleanup incomplete [ns:{}]", self.name) + } else { + format!("Network namespace cleaned up [ns:{}]", self.name) + }) + .build(); + openshell_ocsf::ocsf_emit!(event); } } @@ -597,10 +963,25 @@ impl Drop for NetworkNamespace { /// /// Returns an error if proxy mode is requested but the namespace cannot be /// created (e.g., missing `CAP_NET_ADMIN` / `CAP_SYS_ADMIN` or `iproute2`). -/// Failure to install nftables bypass-detection rules is non-fatal and is -/// reported via OCSF instead. +/// Legacy bypass-rule installation remains best-effort for compatibility. pub fn create_netns_for_proxy( policy: &openshell_core::policy::SandboxPolicy, +) -> Result> { + create_netns(policy, false) +} + +/// Create a proxy namespace whose nftables policy is a mandatory RFC 0012 +/// default-deny ceiling. Unlike the legacy helper, any installation failure +/// aborts boundary establishment. +pub fn create_conformant_netns_for_proxy( + policy: &openshell_core::policy::SandboxPolicy, +) -> Result> { + create_netns(policy, true) +} + +fn create_netns( + policy: &openshell_core::policy::SandboxPolicy, + require_egress_ceiling: bool, ) -> Result> { use openshell_core::policy::NetworkMode; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; @@ -608,7 +989,12 @@ pub fn create_netns_for_proxy( if !matches!(policy.network.mode, NetworkMode::Proxy) { return Ok(None); } - match NetworkNamespace::create() { + let namespace = if require_egress_ceiling { + NetworkNamespace::create_conformant() + } else { + NetworkNamespace::create() + }; + match namespace { Ok(ns) => { let proxy_port = policy .network @@ -616,14 +1002,26 @@ pub fn create_netns_for_proxy( .as_ref() .and_then(|p| p.http_addr) .map_or(3128, |addr| addr.port()); - if let Err(e) = ns.install_bypass_rules(proxy_port) { + if require_egress_ceiling { + ns.install_egress_ceiling(proxy_port).map_err(|error| { + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "failed") + .message(format!("Failed to establish egress ceiling: {error}")) + .build() + ); + error + })?; + } else if let Err(error) = ns.install_bypass_rules(proxy_port) { ocsf_emit!( ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(SeverityId::Medium) .status(StatusId::Failure) .state(StateId::Disabled, "degraded") .message(format!( - "Failed to install bypass detection rules (non-fatal): {e}" + "Failed to install bypass detection rules (non-fatal): {error}" )) .build() ); @@ -666,7 +1064,7 @@ pub fn install_sidecar_bypass_rules(proxy_uid: u32) -> Result<()> { } fn install_sidecar_nft_bypass_rules(proxy_uid: u32) -> Result<()> { - let nft_cmd = find_nft().ok_or_else(|| { + let nft_cmd = find_nft(HelperSource::TrustedSupervisorRuntime).ok_or_else(|| { miette::miette!( "trusted nft helper not found; sidecar network enforcement requires nftables" ) @@ -680,14 +1078,14 @@ const SIDECAR_IPTABLES_CHAIN: &str = "OPENSHELL_SIDECAR_BYPASS"; const PROC_NET_IF_INET6_PATH: &str = "/proc/net/if_inet6"; fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { - let ipv4_filter_tool = find_iptables_legacy().ok_or_else(|| { + let ipv4_filter_tool = find_iptables_legacy(HelperSource::TrustedSupervisorRuntime).ok_or_else(|| { miette::miette!( "trusted iptables-legacy helper not found; sidecar network enforcement fallback unavailable" ) })?; let ipv6_fence_tool = if current_namespace_has_non_loopback_ipv6()? { - Some(find_ip6tables_legacy().ok_or_else(|| { + Some(find_ip6tables_legacy(HelperSource::TrustedSupervisorRuntime).ok_or_else(|| { miette::miette!( "trusted ip6tables-legacy helper not found; sidecar network enforcement fallback cannot fence IPv6" ) @@ -699,17 +1097,14 @@ fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { None }; - cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_deref()); + cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_ref()); if let Err(e) = install_sidecar_iptables_legacy_family_rules( &ipv4_filter_tool, proxy_uid, "icmp-port-unreachable", ) { - cleanup_sidecar_iptables_legacy_rule_families( - &ipv4_filter_tool, - ipv6_fence_tool.as_deref(), - ); + cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_ref()); return Err(e); } @@ -746,7 +1141,7 @@ fn has_non_loopback_ipv6_interface(content: &str) -> bool { } fn install_sidecar_iptables_legacy_family_rules( - cmd: &str, + cmd: &TrustedHelper, proxy_uid: u32, udp_reject_with: &str, ) -> Result<()> { @@ -807,7 +1202,7 @@ fn install_sidecar_iptables_legacy_family_rules( Ok(()) } -fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { +fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &TrustedHelper) { while run_iptables_legacy_current_namespace( iptables_cmd, &["-D", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], @@ -818,28 +1213,70 @@ fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-X", SIDECAR_IPTABLES_CHAIN]); } -fn cleanup_sidecar_iptables_legacy_rule_families(ipv4_cmd: &str, ipv6_cmd: Option<&str>) { +fn cleanup_sidecar_iptables_legacy_rule_families( + ipv4_cmd: &TrustedHelper, + ipv6_cmd: Option<&TrustedHelper>, +) { cleanup_sidecar_iptables_legacy_rules(ipv4_cmd); if let Some(ipv6_cmd) = ipv6_cmd { cleanup_sidecar_iptables_legacy_rules(ipv6_cmd); } } +#[allow(unsafe_code)] +fn trusted_command_in_netns(helper: &TrustedHelper, netns: &str) -> Result { + use std::os::unix::process::CommandExt as _; + + let namespace = std::fs::File::open(format!("/var/run/netns/{netns}")).into_diagnostic()?; + let mut command = helper.command(); + // SAFETY: `setns` is async-signal-safe and the captured file remains open + // in the child until this pre-exec hook completes. + unsafe { + command.pre_exec(move || { + if libc::setns(namespace.as_raw_fd(), libc::CLONE_NEWNET) == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + Ok(command) +} + +#[allow(unsafe_code)] +fn trusted_tokio_command_in_netns( + helper: &TrustedHelper, + netns: &str, +) -> Result { + let namespace = std::fs::File::open(format!("/var/run/netns/{netns}")).into_diagnostic()?; + let mut command = helper.tokio_command(); + // SAFETY: `setns` is async-signal-safe and the captured file remains open + // in the child until this pre-exec hook completes. + unsafe { + command.pre_exec(move || { + if libc::setns(namespace.as_raw_fd(), libc::CLONE_NEWNET) == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + Ok(command) +} + /// Run an `ip` command on the host. -fn run_ip(args: &[&str]) -> Result<()> { - let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; +fn run_ip(source: HelperSource, args: &[&str]) -> Result<()> { + let ip = find_binary(source, "ip", IP_SEARCH_PATHS)?; - debug!(command = %format!("{ip_path} {}", args.join(" ")), "Running ip command"); + debug!(command = %format!("{} {}", ip.executable.display(), args.join(" ")), "Running ip command"); - let output = Command::new(ip_path) - .args(args) - .output() - .into_diagnostic()?; + let output = ip.command().args(args).output().into_diagnostic()?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "{ip_path} {} failed: {}", + "{} {} failed: {}", + ip.executable.display(), args.join(" "), stderr.trim() )); @@ -848,13 +1285,17 @@ fn run_ip(args: &[&str]) -> Result<()> { Ok(()) } -fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> Result<()> { +fn run_iptables_legacy_current_namespace( + iptables_cmd: &TrustedHelper, + args: &[&str], +) -> Result<()> { debug!( - command = %format!("{iptables_cmd} {}", args.join(" ")), + command = %format!("{} {}", iptables_cmd.executable.display(), args.join(" ")), "Running iptables-legacy sidecar command" ); - let output = Command::new(iptables_cmd) + let output = iptables_cmd + .command() .args(args) .output() .into_diagnostic()?; @@ -862,7 +1303,8 @@ fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> R if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "{iptables_cmd} {} failed: {}", + "{} {} failed: {}", + iptables_cmd.executable.display(), args.join(" "), stderr.trim() )); @@ -880,14 +1322,15 @@ fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> R /// Commands marked as non-required are allowed to fail with a warning. /// Required commands that fail abort the sequence immediately. fn run_nft_commands_current_namespace( - nft_cmd: &str, + nft_cmd: &TrustedHelper, commands: &[nft_ruleset::NftCommand], ) -> Result<()> { for cmd in commands { let args_str = cmd.args.join(" "); - debug!(command = %format!("{nft_cmd} {args_str}"), "Running nft command"); + debug!(command = %format!("{} {args_str}", nft_cmd.executable.display()), "Running nft command"); - let output = Command::new(nft_cmd) + let output = nft_cmd + .command() .args(&cmd.args) .output() .into_diagnostic()?; @@ -896,7 +1339,8 @@ fn run_nft_commands_current_namespace( let stderr = String::from_utf8_lossy(&output.stderr); if cmd.required { return Err(miette::miette!( - "{nft_cmd} {args_str} failed: {}", + "{} {args_str} failed: {}", + nft_cmd.executable.display(), stderr.trim() )); } @@ -910,44 +1354,32 @@ fn run_nft_commands_current_namespace( Ok(()) } -/// Run an `ip` command inside a network namespace via `nsenter --net=`. +/// Run an `ip` command inside a network namespace. /// -/// We use `nsenter` instead of `ip netns exec` because `ip netns exec` -/// remounts `/sys` to reflect the target namespace's sysfs entries. That -/// sysfs remount requires real `CAP_SYS_ADMIN` in the host user namespace, -/// which is unavailable in rootless container runtimes (e.g. rootless -/// Podman). `nsenter --net=` enters only the network namespace without -/// changing the mount namespace, avoiding the sysfs remount entirely. -/// The supervisor's operations (addr add, link set, route add) are all -/// netlink-based and do not need sysfs access. -fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { - run_ip_netns_output(netns, args).map(|_| ()) +/// The child enters only the network namespace before exec. This avoids both +/// `ip netns exec`'s sysfs remount and a separate `nsenter` helper. +fn run_ip_netns(source: HelperSource, netns: &str, args: &[&str]) -> Result<()> { + run_ip_netns_output(source, netns, args).map(|_| ()) } -fn run_ip_netns_output(netns: &str, args: &[&str]) -> Result { - let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; - let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; - let ns_path = openshell_core::container_paths::netns_path(netns); - let net_flag = format!("--net={}", ns_path.display()); - - let mut full_args = vec![net_flag.as_str(), "--", ip_path]; - full_args.extend(args); +fn run_ip_netns_output(source: HelperSource, netns: &str, args: &[&str]) -> Result { + let ip = find_binary(source, "ip", IP_SEARCH_PATHS)?; debug!( - command = %format!("{nsenter_path} {}", full_args.join(" ")), - "Running ip in namespace via nsenter" + command = %format!("{} {}", ip.executable.display(), args.join(" ")), + "Running ip in namespace" ); - let output = Command::new(nsenter_path) - .args(&full_args) + let output = trusted_command_in_netns(&ip, netns)? + .args(args) .output() .into_diagnostic()?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "{nsenter_path} --net={} {ip_path} {} failed: {}", - ns_path.display(), + "{} {} failed in netns {netns}: {}", + ip.executable.display(), args.join(" "), stderr.trim() )); @@ -980,32 +1412,25 @@ fn first_route_overlap( }) } -/// Run a sequence of nft commands inside a network namespace via `nsenter --net=`. +/// Run a sequence of nft commands inside a network namespace. /// /// Each command is executed as a separate invocation to avoid atomic batch /// rollback. See [`run_nft_commands_current_namespace`] for rationale. fn run_nft_commands_netns( netns: &str, - nft_cmd: &str, + nft_cmd: &TrustedHelper, commands: &[nft_ruleset::NftCommand], ) -> Result<()> { - let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; - let ns_path = openshell_core::container_paths::netns_path(netns); - let net_flag = format!("--net={}", ns_path.display()); - for cmd in commands { let args_str = cmd.args.join(" "); debug!( - command = %format!("{nsenter_path} {net_flag} -- {nft_cmd} {args_str}"), + command = %format!("{} {args_str}", nft_cmd.executable.display()), "Running nft command in namespace" ); - let mut full_args = vec![net_flag.as_str(), "--", nft_cmd]; let arg_refs: Vec<&str> = cmd.args.iter().map(String::as_str).collect(); - full_args.extend(&arg_refs); - - let output = Command::new(nsenter_path) - .args(&full_args) + let output = trusted_command_in_netns(nft_cmd, netns)? + .args(&arg_refs) .output() .into_diagnostic()?; @@ -1055,109 +1480,282 @@ fn enable_nf_log_all_netns() { } } -/// Well-known paths where nft may be installed. -const NFT_SEARCH_PATHS: &[&str] = &["/usr/sbin/nft", "/sbin/nft", "/usr/bin/nft"]; +/// Paths within the driver-controlled supervisor runtime. +const NFT_SEARCH_PATHS: &[&str] = &["usr/sbin/nft", "sbin/nft", "usr/bin/nft"]; const IPTABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ - "/usr/sbin/iptables-legacy", - "/sbin/iptables-legacy", - "/usr/bin/iptables-legacy", + "usr/sbin/iptables-legacy", + "sbin/iptables-legacy", + "usr/bin/iptables-legacy", ]; const IP6TABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ - "/usr/sbin/ip6tables-legacy", - "/sbin/ip6tables-legacy", - "/usr/bin/ip6tables-legacy", + "usr/sbin/ip6tables-legacy", + "sbin/ip6tables-legacy", + "usr/bin/ip6tables-legacy", ]; -fn find_trusted_binary<'a>(name: &str, paths: &'a [&str]) -> Result<&'a str> { - paths +fn trusted_runtime_root() -> Result { + #[cfg(test)] + if let Some(root) = std::env::var_os("OPENSHELL_TEST_TRUSTED_RUNTIME_ROOT") { + return Ok(PathBuf::from(root)); + } + if let Some(root) = TRUSTED_RUNTIME_ROOT.get() { + return Ok(root.clone()); + } + let executable = std::env::current_exe().into_diagnostic()?; + let parent = executable + .parent() + .ok_or_else(|| miette::miette!("supervisor executable has no parent directory"))?; + Ok(parent.join("openshell-runtime")) +} + +fn find_binary(source: HelperSource, name: &str, paths: &[&str]) -> Result { + match source { + HelperSource::LegacyWorkloadImage => find_legacy_binary(name, paths), + HelperSource::TrustedSupervisorRuntime => find_trusted_binary(name, paths), + } +} + +fn find_legacy_binary(name: &str, paths: &[&str]) -> Result { + use std::os::unix::fs::MetadataExt as _; + + let trusted_uid = nix::unistd::geteuid().as_raw(); + let executable = paths + .iter() + .map(|path| Path::new("/").join(path)) + .find_map(|path| { + let resolved = path.canonicalize().ok()?; + let metadata = resolved.metadata().ok()?; + (metadata.is_file() + && metadata.uid() == trusted_uid + && metadata.mode() & 0o111 != 0 + && metadata.mode() & 0o022 == 0) + .then_some(resolved) + }) + .ok_or_else(|| { + miette::miette!( + "{name} helper not found in legacy workload image; checked {}", + paths.join(", ") + ) + })?; + Ok(TrustedHelper { + executable, + loader: None, + library_path: String::new(), + xtables_path: PathBuf::new(), + }) +} + +fn find_trusted_binary(name: &str, paths: &[&str]) -> Result { + find_trusted_binary_in(&trusted_runtime_root()?, name, paths) +} + +fn find_trusted_binary_in(root: &Path, name: &str, paths: &[&str]) -> Result { + use std::os::unix::fs::MetadataExt; + + let trusted_uid = nix::unistd::geteuid().as_raw(); + let resolved_root = root.canonicalize().map_err(|error| { + miette::miette!( + "trusted supervisor helper runtime {} is unavailable: {error}", + root.display() + ) + })?; + // Kubernetes and Podman preserve root ownership from the supervisor image. + // Docker may materialize the same image-owned runtime in a gateway-user + // cache before bind-mounting it read-only. In that case the immutable + // mount, not its namespace-visible UID, establishes provenance. + let runtime_is_read_only = nix::sys::statvfs::statvfs(&resolved_root) + .is_ok_and(|stat| stat.flags().contains(nix::sys::statvfs::FsFlags::ST_RDONLY)); + let executable = paths .iter() - .copied() - .find(|path| { - let path = Path::new(path); - path.is_absolute() && path.is_file() + .map(|path| resolved_root.join(path)) + .find_map(|path| { + let resolved = path.canonicalize().ok()?; + if !resolved.starts_with(&resolved_root) { + return None; + } + let Ok(metadata) = resolved.metadata() else { + return None; + }; + (metadata.is_file() + && (metadata.uid() == trusted_uid || runtime_is_read_only) + && metadata.mode() & 0o111 != 0 + && metadata.mode() & 0o022 == 0) + .then_some(resolved) }) .ok_or_else(|| { miette::miette!( - "trusted {name} helper not found; checked {}", + "trusted {name} helper not found below {}; checked {}", + resolved_root.display(), paths.join(", ") ) + })?; + let loader = runtime_library_directories(&resolved_root) + .into_iter() + .filter_map(|directory| std::fs::read_dir(directory).ok()) + .flatten() + .filter_map(std::result::Result::ok) + .map(|entry| entry.path()) + .find(|path| is_runtime_loader(path)) + .ok_or_else(|| { + miette::miette!( + "trusted dynamic loader not found below {}", + resolved_root.display() + ) + })? + .canonicalize() + .into_diagnostic()?; + if !loader.starts_with(&resolved_root) { + return Err(miette::miette!("trusted runtime loader escapes its root")); + } + let loader_metadata = loader.metadata().into_diagnostic()?; + if !loader_metadata.is_file() + || (loader_metadata.uid() != trusted_uid && !runtime_is_read_only) + || loader_metadata.mode() & 0o111 == 0 + || loader_metadata.mode() & 0o022 != 0 + { + return Err(miette::miette!( + "trusted runtime loader has unsafe ownership or mode" + )); + } + let library_path = runtime_library_directories(&resolved_root) + .into_iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(":"); + let xtables_path = runtime_library_directories(&resolved_root) + .into_iter() + .map(|directory| directory.join("xtables")) + .find(|path| path.is_dir()) + .unwrap_or_else(|| resolved_root.join("usr/lib/xtables")); + Ok(TrustedHelper { + executable, + loader: Some(loader), + library_path, + xtables_path, + }) +} + +fn runtime_library_directories(root: &Path) -> Vec { + let mut directories = Vec::new(); + for base in ["lib", "lib64", "usr/lib", "usr/lib64"] { + let base = root.join(base); + if !base.is_dir() { + continue; + } + directories.push(base.clone()); + if let Ok(entries) = std::fs::read_dir(base) { + directories.extend( + entries + .filter_map(std::result::Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()), + ); + } + } + directories +} + +fn is_runtime_loader(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + (name.starts_with("ld-musl-") && name.ends_with(".so.1")) + || name == "ld-linux-x86-64.so.2" + || name == "ld-linux-aarch64.so.1" }) } /// Find the nft binary path, checking well-known locations. -fn find_nft() -> Option { - find_trusted_binary("nft", NFT_SEARCH_PATHS) - .ok() - .map(String::from) +fn find_nft(source: HelperSource) -> Option { + find_binary(source, "nft", NFT_SEARCH_PATHS).ok() } -fn find_iptables_legacy() -> Option { - find_trusted_binary("iptables-legacy", IPTABLES_LEGACY_SEARCH_PATHS) - .ok() - .map(String::from) +fn find_iptables_legacy(source: HelperSource) -> Option { + find_binary(source, "iptables-legacy", IPTABLES_LEGACY_SEARCH_PATHS).ok() } -fn find_ip6tables_legacy() -> Option { - find_trusted_binary("ip6tables-legacy", IP6TABLES_LEGACY_SEARCH_PATHS) - .ok() - .map(String::from) +fn find_ip6tables_legacy(source: HelperSource) -> Option { + find_binary(source, "ip6tables-legacy", IP6TABLES_LEGACY_SEARCH_PATHS).ok() } #[cfg(test)] mod tests { use super::*; use std::fs; + use std::os::unix::fs::PermissionsExt as _; // These tests require root and network namespace support // Run with: sudo cargo test -- --ignored #[test] - fn find_trusted_binary_uses_absolute_existing_file() { + fn find_trusted_binary_uses_only_the_supplied_runtime() { let tempdir = tempfile::tempdir().unwrap(); - let helper = tempdir.path().join("ip"); + let helper = tempdir.path().join("usr/sbin/ip"); + fs::create_dir_all(helper.parent().unwrap()).unwrap(); fs::write(&helper, b"test helper").unwrap(); - let helper = helper.to_str().unwrap(); - - assert_eq!( - find_trusted_binary("ip", &["relative-ip", "/missing/ip", helper]).unwrap(), - helper - ); + fs::set_permissions(&helper, fs::Permissions::from_mode(0o755)).unwrap(); + let lib = tempdir.path().join("lib"); + fs::create_dir(&lib).unwrap(); + let loader = lib.join("ld-musl-test.so.1"); + fs::write(&loader, b"test loader").unwrap(); + fs::set_permissions(loader, fs::Permissions::from_mode(0o755)).unwrap(); + + let resolved = find_trusted_binary_in(tempdir.path(), "ip", &["usr/sbin/ip"]).unwrap(); + assert_eq!(resolved.executable, helper); } #[test] fn find_trusted_binary_rejects_missing_helpers() { - let err = - find_trusted_binary("nsenter", &["relative-nsenter", "/missing/nsenter"]).unwrap_err(); + let tempdir = tempfile::tempdir().unwrap(); + let err = find_trusted_binary_in(tempdir.path(), "ip", &["usr/sbin/ip"]).unwrap_err(); - assert!(err.to_string().contains("trusted nsenter helper not found")); + assert!(err.to_string().contains("trusted ip helper not found")); + } + + #[test] + fn trusted_runtime_rejects_helper_symlink_into_workload_root() { + use std::os::unix::fs::symlink; + + let runtime = tempfile::tempdir().unwrap(); + let workload = tempfile::tempdir().unwrap(); + let malicious = workload.path().join("ip"); + fs::write(&malicious, b"malicious workload helper").unwrap(); + fs::set_permissions(&malicious, fs::Permissions::from_mode(0o755)).unwrap(); + let helper = runtime.path().join("usr/sbin/ip"); + fs::create_dir_all(helper.parent().unwrap()).unwrap(); + symlink(&malicious, &helper).unwrap(); + + let error = find_trusted_binary_in(runtime.path(), "ip", &["usr/sbin/ip"]) + .expect_err("helper escaping the trusted runtime must be rejected"); + assert!(error.to_string().contains("trusted ip helper not found")); } #[test] - fn nft_search_paths_are_absolute() { + fn nft_search_paths_are_runtime_relative() { for path in NFT_SEARCH_PATHS { assert!( - path.starts_with('/'), - "NFT_SEARCH_PATHS entry must be absolute: {path}" + !path.starts_with('/'), + "NFT_SEARCH_PATHS entry must be runtime-relative: {path}" ); } } #[test] - fn iptables_legacy_search_paths_are_absolute() { + fn iptables_legacy_search_paths_are_runtime_relative() { for path in IPTABLES_LEGACY_SEARCH_PATHS { assert!( - path.starts_with('/'), - "IPTABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" + !path.starts_with('/'), + "IPTABLES_LEGACY_SEARCH_PATHS entry must be runtime-relative: {path}" ); } } #[test] - fn ip6tables_legacy_search_paths_are_absolute() { + fn ip6tables_legacy_search_paths_are_runtime_relative() { for path in IP6TABLES_LEGACY_SEARCH_PATHS { assert!( - path.starts_with('/'), - "IP6TABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" + !path.starts_with('/'), + "IP6TABLES_LEGACY_SEARCH_PATHS entry must be runtime-relative: {path}" ); } } @@ -1186,25 +1784,112 @@ fe800000000000000000000000000001 02 40 20 80 eth0 } #[test] - fn route_overlap_detects_reserved_pool_collision() { - let reserved = [ - "198.18.1.0/25".parse().unwrap(), - "fd23:6f70:656e:1::/120".parse().unwrap(), - ]; - let routes = "default via 10.200.0.1 dev veth\n198.18.0.0/15 dev eth1\n"; - let (route, pool) = first_route_overlap(routes, &reserved).expect("collision"); - assert_eq!(route.to_string(), "198.18.0.0/15"); - assert_eq!(pool.to_string(), "198.18.1.0/25"); + fn egress_ceiling_verification_accepts_required_live_rules() { + let ruleset = br#"{ + "nftables": [ + {"chain":{"family":"inet","table":"openshell_bypass","name":"output","type":"filter","hook":"output","prio":0,"policy":"drop"}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {"match":{"op":"==","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":"10.200.0.1"}}, + {"match":{"op":"==","left":{"payload":{"protocol":"tcp","field":"dport"}},"right":3128}}, + {"accept":null} + ]}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {"match":{"op":"==","left":{"meta":{"key":"oifname"}},"right":"lo"}}, + {"accept":null} + ]}} + ] + }"#; + + verify_egress_ceiling_json(ruleset, "10.200.0.1", 3128).unwrap(); } #[test] - fn route_overlap_ignores_default_and_unrelated_routes() { - let reserved = [ - "198.18.1.0/25".parse().unwrap(), - "fd23:6f70:656e:1::/120".parse().unwrap(), - ]; - let routes = "default via 10.200.0.1 dev veth\n10.200.0.0/24 dev veth\n"; - assert_eq!(first_route_overlap(routes, &reserved), None); + fn egress_ceiling_verification_rejects_fail_open_chain() { + let ruleset = br#"{ + "nftables": [ + {"chain":{"family":"inet","table":"openshell_bypass","name":"output","hook":"output","policy":"accept"}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[{"match":{"right":"10.200.0.1"}},{"match":{"right":3128}},{"accept":null}]}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[{"match":{"right":"lo"}},{"accept":null}]}} + ] + }"#; + + assert!(verify_egress_ceiling_json(ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_missing_required_allow() { + let ruleset = br#"{ + "nftables": [ + {"chain":{"family":"inet","table":"openshell_bypass","name":"output","hook":"output","policy":"drop"}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[{"match":{"right":"lo"}},{"accept":null}]}} + ] + }"#; + + assert!(verify_egress_ceiling_json(ruleset, "10.200.0.1", 3128).is_err()); + } + + fn ruleset_with_accept_expressions(expressions: &str) -> Vec { + format!( + r#"{{"nftables":[ + {{"chain":{{"family":"inet","table":"openshell_bypass","name":"output","type":"filter","hook":"output","prio":0,"policy":"drop"}}}}, + {{"rule":{{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {{"match":{{"op":"==","left":{{"payload":{{"protocol":"ip","field":"daddr"}}}},"right":"10.200.0.1"}}}}, + {{"match":{{"op":"==","left":{{"payload":{{"protocol":"tcp","field":"dport"}}}},"right":3128}}}},{{"accept":null}}]}}}}, + {{"rule":{{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {{"match":{{"op":"==","left":{{"meta":{{"key":"oifname"}}}},"right":"lo"}}}},{{"accept":null}}]}}}}, + {{"rule":{{"family":"inet","table":"openshell_bypass","chain":"output","expr":{expressions}}}}} + ]}}"# + ) + .into_bytes() + } + + #[test] + fn egress_ceiling_verification_rejects_unconditional_accept() { + let ruleset = ruleset_with_accept_expressions(r#"[{"accept":null}]"#); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_unrelated_matching_metadata() { + let ruleset = ruleset_with_accept_expressions( + r#"[{"comment":{"address":"10.200.0.1","port":3128}},{"accept":null}]"#, + ); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_wrong_protocol_or_operator() { + for expressions in [ + r#"[{"match":{"op":"==","left":{"payload":{"protocol":"udp","field":"dport"}},"right":3128}},{"accept":null}]"#, + r#"[{"match":{"op":"!=","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":"10.200.0.1"}},{"accept":null}]"#, + ] { + let ruleset = ruleset_with_accept_expressions(expressions); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + } + + #[test] + fn egress_ceiling_verification_rejects_extra_destination_allow() { + let ruleset = ruleset_with_accept_expressions( + r#"[{"match":{"op":"==","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":"203.0.113.1"}},{"accept":null}]"#, + ); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_jump_to_unverified_chain() { + let ruleset = ruleset_with_accept_expressions(r#"[{"jump":{"target":"unverified"}}]"#); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_ignores_accept_in_another_chain() { + let mut document: serde_json::Value = + serde_json::from_slice(&ruleset_with_accept_expressions(r#"[{"accept":null}]"#)) + .unwrap(); + document["nftables"][3]["rule"]["chain"] = serde_json::json!("other"); + let ruleset = serde_json::to_vec(&document).unwrap(); + verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).unwrap(); } #[test] @@ -1214,8 +1899,8 @@ fe800000000000000000000000000001 02 40 20 80 eth0 let name = ns.name().to_string(); // Verify namespace exists - let ns_path = openshell_core::container_paths::netns_path(&name); - assert!(ns_path.exists(), "Namespace file should exist"); + let ns_path = format!("/var/run/netns/{name}"); + assert!(Path::new(&ns_path).exists(), "Namespace file should exist"); // Verify IPs are set correctly assert_eq!( @@ -1236,4 +1921,181 @@ fe800000000000000000000000000001 02 40 20 80 eth0 "Namespace should be cleaned up" ); } + + #[test] + #[ignore = "requires root privileges"] + fn installed_egress_ceiling_round_trips_through_kernel() { + let ns = NetworkNamespace::create_conformant().expect("create conformant namespace"); + ns.install_egress_ceiling(3128).expect("install ceiling"); + ns.verify_egress_ceiling(3128).expect("verify ceiling"); + } + + #[test] + #[ignore = "requires root privileges"] + fn installed_egress_ceiling_allows_only_proxy_tcp() { + use std::time::Duration; + + #[allow(unsafe_code)] + fn enter_namespace(ns_fd: RawFd) { + // SAFETY: the owning NetworkNamespace remains alive until every + // test thread has joined, so the descriptor stays valid. + let result = unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) }; + assert_eq!(result, 0, "enter workload network namespace"); + } + + let ns = NetworkNamespace::create_conformant().expect("create conformant namespace"); + let ns_fd = ns.ns_fd().expect("network namespace fd"); + let host_ip = ns.host_ip(); + + let alternate_host_ip: std::net::Ipv4Addr = "10.200.0.3".parse().unwrap(); + run_ip( + ns.helper_source, + &["addr", "add", "10.200.0.3/24", "dev", &ns.veth_host], + ) + .expect("add alternate routed IPv4 destination"); + let host_ipv6: std::net::Ipv6Addr = "fd00:200::1".parse().unwrap(); + run_ip( + ns.helper_source, + &[ + "-6", + "addr", + "add", + "fd00:200::1/64", + "dev", + &ns.veth_host, + "nodad", + ], + ) + .expect("add host IPv6 destination"); + run_ip_netns( + ns.helper_source, + ns.name(), + &[ + "-6", + "addr", + "add", + "fd00:200::2/64", + "dev", + &ns.veth_sandbox, + "nodad", + ], + ) + .expect("add workload IPv6 source"); + + // Positive controls prove each route/protocol works before the ceiling + // is installed, so later denial cannot pass because of broken setup. + let ipv4_control = + std::net::TcpListener::bind((alternate_host_ip, 0)).expect("bind IPv4 control"); + let ipv4_control_address = ipv4_control.local_addr().unwrap(); + assert!( + std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&ipv4_control_address, Duration::from_secs(1)) + }) + .join() + .expect("IPv4 control thread") + .is_ok(), + "alternate IPv4 route must work before enforcement" + ); + + let ipv6_control = std::net::TcpListener::bind((host_ipv6, 0)).expect("bind IPv6 control"); + let ipv6_control_address = ipv6_control.local_addr().unwrap(); + assert!( + std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&ipv6_control_address, Duration::from_secs(1)) + }) + .join() + .expect("IPv6 control thread") + .is_ok(), + "IPv6 route must work before enforcement" + ); + + let udp_control = + std::net::UdpSocket::bind((host_ip, 0)).expect("bind UDP positive control"); + udp_control + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let udp_control_address = udp_control.local_addr().unwrap(); + std::thread::spawn(move || { + enter_namespace(ns_fd); + let socket = std::net::UdpSocket::bind("0.0.0.0:0").expect("bind control UDP"); + socket.send_to(b"control", udp_control_address) + }) + .join() + .expect("UDP control thread") + .expect("send UDP positive control"); + let mut control = [0_u8; 7]; + udp_control + .recv_from(&mut control) + .expect("UDP route must work before enforcement"); + assert_eq!(&control, b"control"); + + let proxy = std::net::TcpListener::bind((host_ip, 0)).expect("bind proxy listener"); + let proxy_address = proxy.local_addr().expect("proxy address"); + ns.install_egress_ceiling(proxy_address.port()) + .expect("install ceiling"); + + let allowed = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&proxy_address, Duration::from_secs(1)) + }); + proxy + .set_nonblocking(true) + .expect("set proxy listener nonblocking"); + assert!(allowed.join().expect("allowed-connect thread").is_ok()); + + let direct = std::net::TcpListener::bind((host_ip, 0)).expect("bind direct listener"); + let direct_address = direct.local_addr().expect("direct address"); + let denied = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&direct_address, Duration::from_millis(300)) + }); + assert!( + denied.join().expect("denied-connect thread").is_err(), + "direct TCP must not bypass mediation" + ); + + let alternate = std::net::TcpListener::bind((alternate_host_ip, proxy_address.port())) + .expect("bind alternate routed listener"); + let alternate_address = alternate.local_addr().unwrap(); + let denied = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&alternate_address, Duration::from_millis(300)) + }); + assert!( + denied.join().expect("alternate-connect thread").is_err(), + "the proxy port at another routed IPv4 destination must be denied" + ); + + let ipv6 = std::net::TcpListener::bind((host_ipv6, proxy_address.port())) + .expect("bind IPv6 observer"); + let ipv6_address = ipv6.local_addr().unwrap(); + let denied = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&ipv6_address, Duration::from_millis(300)) + }); + assert!( + denied.join().expect("IPv6-connect thread").is_err(), + "direct IPv6 TCP must not bypass mediation" + ); + + let udp = std::net::UdpSocket::bind((host_ip, proxy_address.port())) + .expect("bind UDP observer at proxy destination"); + udp.set_read_timeout(Some(Duration::from_millis(300))) + .expect("set UDP timeout"); + let udp_address = udp.local_addr().expect("UDP address"); + let udp_send = std::thread::spawn(move || { + enter_namespace(ns_fd); + let socket = std::net::UdpSocket::bind("0.0.0.0:0").expect("bind workload UDP"); + socket.send_to(b"bypass", udp_address) + }) + .join() + .expect("UDP thread"); + let mut byte = [0_u8; 1]; + assert!( + udp_send.is_err() || udp.recv_from(&mut byte).is_err(), + "direct UDP must not bypass mediation" + ); + } } diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index cb1399d830..b417440906 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -29,8 +29,10 @@ use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::path::Path; use std::path::PathBuf; use std::process::Stdio; +use std::sync::Arc; #[cfg(target_os = "linux")] use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, Ordering}; #[cfg(target_os = "linux")] use std::sync::mpsc; use tokio::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; @@ -145,19 +147,50 @@ pub(crate) fn prepare_child_sandbox( policy: &SandboxPolicy, workdir: Option<&str>, enforcement_mode: ProcessEnforcementMode, + runtime_read_only: &[PathBuf], ) -> Result> { if !enforcement_mode.enforces_child_sandbox() { return Ok(None); } + let effective_policy = policy_with_runtime_read_only(policy, runtime_read_only); + let prepared = if enforcement_mode.uses_privileged_process_setup() { - sandbox::linux::prepare(policy, workdir) + sandbox::linux::prepare(&effective_policy, workdir) } else { - sandbox::linux::prepare_current_user(policy, workdir) + sandbox::linux::prepare_current_user(&effective_policy, workdir) }?; Ok(Some(prepared)) } +#[cfg(target_os = "linux")] +fn policy_with_runtime_read_only( + policy: &SandboxPolicy, + runtime_read_only: &[PathBuf], +) -> SandboxPolicy { + let mut effective_policy = policy.clone(); + for path in runtime_read_only { + if !effective_policy.filesystem.read_only.contains(path) { + effective_policy.filesystem.read_only.push(path.clone()); + } + } + effective_policy +} + +#[cfg(target_os = "linux")] +pub(crate) fn ca_runtime_read_only_paths(ca_paths: Option<&(PathBuf, PathBuf)>) -> Vec { + let Some((certificate, bundle)) = ca_paths else { + return Vec::new(); + }; + let mut paths = Vec::with_capacity(3); + if let Some(directory) = certificate.parent() { + paths.push(directory.to_path_buf()); + } + paths.push(certificate.clone()); + paths.push(bundle.clone()); + paths +} + const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::OCI_IMAGE_USER, openshell_core::sandbox_env::SANDBOX_UID, @@ -656,6 +689,8 @@ pub struct ProcessHandle { child: Child, pid: u32, io: Option, + terminal: Arc, + signal_lock: Arc>, #[cfg(target_os = "linux")] managed_child: Option, } @@ -842,8 +877,14 @@ impl ProcessHandle { // runs as the sandbox UID, so inaccessible paths are unavailable to // the workload and best-effort compatibility skips them. #[cfg(target_os = "linux")] - let prepared_sandbox = prepare_child_sandbox(policy, workspace.root(), enforcement_mode) - .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; + let runtime_read_only = ca_runtime_read_only_paths(ca_paths); + let prepared_sandbox = prepare_child_sandbox( + policy, + workspace.root(), + enforcement_mode, + &runtime_read_only, + ) + .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; // Set up process group for signal handling (non-interactive mode only). // In interactive mode, we inherit the parent's process group to maintain // proper terminal control for shells and interactive programs. @@ -930,6 +971,8 @@ impl ProcessHandle { child, pid, io: Some(io), + terminal: Arc::new(AtomicBool::new(false)), + signal_lock: Arc::new(std::sync::Mutex::new(())), #[cfg(target_os = "linux")] managed_child, }) @@ -1078,6 +1121,8 @@ impl ProcessHandle { child, pid, io: Some(io), + terminal: Arc::new(AtomicBool::new(false)), + signal_lock: Arc::new(std::sync::Mutex::new(())), }) } @@ -1092,6 +1137,12 @@ impl ProcessHandle { self.io.take().expect("canonical process I/O already taken") } + /// Shared state used by an independent boundary signal handle. + #[must_use] + pub fn signaling_state(&self) -> (Arc, Arc>) { + (self.terminal.clone(), self.signal_lock.clone()) + } + /// Wait for the process to exit. /// /// # Errors @@ -1099,6 +1150,11 @@ impl ProcessHandle { /// Returns an error if waiting fails. pub async fn wait(&mut self) -> std::io::Result { let status = self.child.wait().await; + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.terminal.store(true, Ordering::Release); #[cfg(target_os = "linux")] if let Some(child) = self.managed_child.take() { managed_children::unregister(child); @@ -1111,6 +1167,11 @@ impl ProcessHandle { pub fn try_wait(&mut self) -> std::io::Result> { let status = self.child.try_wait()?; if status.is_some() { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.terminal.store(true, Ordering::Release); #[cfg(target_os = "linux")] if let Some(child) = self.managed_child.take() { managed_children::unregister(child); @@ -1125,6 +1186,13 @@ impl ProcessHandle { /// /// Returns an error if the signal cannot be sent. pub fn signal(&self, sig: Signal) -> Result<()> { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("process has exited")); + } let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); signal::kill(Pid::from_raw(pid), sig).into_diagnostic() } @@ -3429,6 +3497,81 @@ mod tests { } } + #[cfg(target_os = "linux")] + #[test] + fn runtime_ca_paths_are_added_to_the_effective_read_only_policy() { + let mut policy = policy_with_process(ProcessPolicy::default()); + policy.filesystem.read_only = vec![PathBuf::from("/usr")]; + let certificate = PathBuf::from("/run/openshell-proxy-ca/ca.crt"); + let bundle = PathBuf::from("/run/openshell-proxy-ca/ca-bundle.crt"); + + let effective = policy_with_runtime_read_only( + &policy, + &[certificate.clone(), bundle.clone(), certificate.clone()], + ); + + assert_eq!(policy.filesystem.read_only, vec![PathBuf::from("/usr")]); + assert_eq!( + effective.filesystem.read_only, + vec![PathBuf::from("/usr"), certificate, bundle] + ); + } + + #[cfg(target_os = "linux")] + #[test] + #[allow(unsafe_code)] + fn runtime_ca_material_remains_readable_after_landlock_for_non_root_workload() { + let root = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let ca_directory = root.path().join("openshell-proxy-ca"); + std::fs::create_dir(&ca_directory).unwrap(); + std::fs::set_permissions(&ca_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let certificate = ca_directory.join("ca.crt"); + let bundle = ca_directory.join("ca-bundle.crt"); + let denied = root.path().join("not-authorized"); + for path in [&certificate, &bundle, &denied] { + std::fs::write(path, b"public certificate material").unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o444)).unwrap(); + } + + let mut policy = policy_with_process(ProcessPolicy::default()); + policy.landlock = LandlockPolicy { + compatibility: openshell_core::policy::LandlockCompatibility::HardRequirement, + }; + let runtime_paths = + ca_runtime_read_only_paths(Some(&(certificate.clone(), bundle.clone()))); + let Ok(Some(prepared)) = + prepare_child_sandbox(&policy, None, ProcessEnforcementMode::Full, &runtime_paths) + else { + return; + }; + + match unsafe { fork() }.expect("fork should succeed") { + ForkResult::Child => { + let dropped = if nix::unistd::geteuid().is_root() { + unsafe { + libc::setgroups(0, std::ptr::null()) == 0 + && libc::setgid(42_235) == 0 + && libc::setuid(42_234) == 0 + } + } else { + true + }; + let valid = dropped + && sandbox::linux::enforce(prepared).is_ok() + && std::fs::read(&certificate).is_ok() + && std::fs::read(&bundle).is_ok() + && std::fs::read(&denied).is_err(); + unsafe { libc::_exit(i32::from(!valid)) }; + } + ForkResult::Parent { child } => assert_eq!( + waitpid(child, None).expect("waitpid should succeed"), + WaitStatus::Exited(child, 0), + "Landlock must preserve non-root access only to admitted public CA material" + ), + } + } + #[cfg(unix)] #[test] fn validate_oci_workspace_rejects_restrictive_parent() { diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index 8c47e789ba..a5f6bf58b2 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -26,6 +26,7 @@ use crate::netns::NetworkNamespace; use openshell_core::policy::{NetworkMode, SandboxPolicy}; use openshell_core::proposals::AgentProposals; use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward}; #[cfg(target_os = "linux")] use openshell_core::activity::ActivitySender; @@ -237,11 +238,6 @@ pub async fn run_process( // CLONE_NEWNET) so it lands inside the workload's network namespace. // Without this, SSH-spawned shells run in the host namespace and bypass // the proxy entirely. - #[cfg(target_os = "linux")] - let ssh_netns_fd = netns.and_then(NetworkNamespace::ns_fd); - #[cfg(not(target_os = "linux"))] - let ssh_netns_fd: Option = None; - #[cfg(target_os = "linux")] let mut handle = ProcessHandle::spawn( program, @@ -282,20 +278,45 @@ pub async fn run_process( #[cfg(not(target_os = "linux"))] let ssh_proxy_url = ssh_proxy_url_for_policy(policy, None); + #[cfg(target_os = "linux")] + let boundary_netns_fd = netns + .map(NetworkNamespace::try_clone_ns_fd) + .transpose()? + .flatten() + .map(Arc::new); + #[cfg(not(target_os = "linux"))] + let boundary_netns_fd = None; + let boundary_runtime = crate::boundary_io::BoundaryRuntimeState::new(); + let user_environment: std::collections::HashMap = + std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) + .ok() + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_default(); + let port_forward: Arc = + Arc::new(crate::boundary_io::NetnsPortForward::new( + boundary_netns_fd.clone(), + Some(boundary_runtime.clone()), + )); + let boundary_exec: Arc = + Arc::new(crate::boundary_exec::LocalBoundaryExec::new( + policy.clone(), + workspace.owned_root(), + boundary_netns_fd, + ssh_proxy_url.clone(), + ca_file_paths.clone().map(Arc::new), + provider_credentials.clone(), + user_environment, + resolved_process_identity, + enforcement_mode, + boundary_runtime.clone(), + )); + let ssh_socket_path: Option = ssh_socket_path.map(std::path::PathBuf::from); if let Some(listen_path) = ssh_socket_path.clone() { - let policy_clone = policy.clone(); - let workspace_clone = workspace.clone(); - let proxy_url = ssh_proxy_url; - let netns_fd = ssh_netns_fd; let ca_paths = ca_file_paths.clone(); - let provider_credentials_clone = provider_credentials.clone(); - let main_session_clone = Arc::clone(&main_session); - let user_env_clone: std::collections::HashMap = - std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) - .ok() - .and_then(|json| serde_json::from_str(&json).ok()) - .unwrap_or_default(); + let ssh_port_forward = port_forward.clone(); + let ssh_boundary_exec = boundary_exec.clone(); + let ssh_main_session = Arc::clone(&main_session); let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); @@ -304,17 +325,12 @@ pub async fn run_process( if let Err(err) = crate::ssh::run_ssh_server( listen_path, ssh_ready_tx, - policy_clone, - workspace_clone, - netns_fd, - proxy_url, ca_paths, - provider_credentials_clone, - user_env_clone, - resolved_process_identity, enforcement_mode, shared_ssh_socket, - main_session_clone, + ssh_port_forward, + ssh_boundary_exec, + Some(ssh_main_session), ) .await { @@ -374,7 +390,7 @@ pub async fn run_process( endpoint.to_string(), id.to_string(), socket.clone(), - ssh_netns_fd, + port_forward, None, Arc::clone(&supervisor_terminating), main_instance_id.clone(), @@ -484,6 +500,7 @@ pub async fn run_process( } supervisor_terminating.store(true, Ordering::Release); + boundary_runtime.deactivate(); if let Some(task) = supervisor_session_task { task.abort(); } @@ -718,7 +735,7 @@ fn ssh_proxy_url_for_policy( /// /// Best-effort: any failure (no gateway, RPC error, install failure) is /// logged but does not fail sandbox startup. -async fn install_initial_agent_skill( +pub(crate) async fn install_initial_agent_skill( sandbox_id: Option<&str>, openshell_endpoint: Option<&str>, agent_proposals: &AgentProposals, diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 6d45e31777..d06903ad0b 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -8,8 +8,8 @@ use crate::main_session::{MainOutput, MainSession}; #[cfg(target_os = "linux")] use crate::managed_children; use crate::process::{ - ProcessEnforcementMode, ResolvedProcessIdentity, ResolvedWorkspace, - drop_privileges_with_identity, is_supervisor_only_env_var, session_user_and_home, + ProcessEnforcementMode, ResolvedProcessIdentity, drop_privileges_with_identity, + is_supervisor_only_env_var, }; use crate::sandbox; #[cfg(unix)] @@ -18,9 +18,10 @@ use miette::{IntoDiagnostic, Result}; use nix::pty::{Winsize, openpty}; use nix::unistd::setsid; use openshell_core::VERSION; +use openshell_core::net::connect_tcp_nodelay_best_effort; +#[cfg(target_os = "linux")] use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::policy::SandboxPolicy; -use openshell_core::provider_credentials::ProviderCredentialState; use openshell_ocsf::{ ActionId, ActivityId, DispositionId, SeverityId, SshActivityBuilder, StatusId, ocsf_emit, }; @@ -30,7 +31,7 @@ use russh::{ChannelId, ChannelOpenFailure, Sig}; use std::borrow::Cow; use std::collections::HashMap; use std::io::{Read, Write}; -use std::os::fd::{AsRawFd, RawFd}; +use std::os::fd::{AsRawFd, OwnedFd, RawFd}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::{Arc, mpsc}; @@ -39,6 +40,28 @@ use tokio::net::UnixListener; use tracing::warn; const NO_LOGIN_SHELL_ENV: (&str, &str) = ("OPENSHELL_NO_LOGIN_SHELL", "1"); +const MAIN_DETACH_PREFIX: u8 = 0x10; +const MAIN_DETACH_KEY: u8 = 0x11; + +fn filter_main_detach_sequence(prefix_pending: &mut bool, data: &[u8]) -> (Vec, bool) { + let mut forward = Vec::with_capacity(data.len() + usize::from(*prefix_pending)); + for &byte in data { + if *prefix_pending { + if byte == MAIN_DETACH_KEY { + *prefix_pending = false; + return (forward, true); + } + forward.push(MAIN_DETACH_PREFIX); + *prefix_pending = false; + } + if byte == MAIN_DETACH_PREFIX { + *prefix_pending = true; + } else { + forward.push(byte); + } + } + (forward, false) +} /// Perform SSH server initialization: generate a host key, build the config, /// and bind the Unix socket listener. Extracted so that startup errors can be @@ -58,7 +81,6 @@ fn ssh_server_init( let mut rng = rand::rng(); let host_key = PrivateKey::random(&mut rng, Algorithm::Ed25519).into_diagnostic()?; - // TODO: while building the SSH config, refactor the server_id to be "SSH-2.0-OpenShell_" from `openshell_core::VERSION` let mut config = russh::server::Config { server_id: russh::SshId::Standard(Cow::Owned(format!("SSH-2.0-OpenShell_{VERSION}"))), auth_rejection_time: Duration::from_secs(1), @@ -119,19 +141,14 @@ fn ssh_server_init( pub async fn run_ssh_server( listen_path: PathBuf, ready_tx: tokio::sync::oneshot::Sender>, - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, ca_file_paths: Option<(PathBuf, PathBuf)>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, shared_socket: bool, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Result<()> { - let (listener, config, ca_paths) = match ssh_server_init( + let (listener, config, _ca_paths) = match ssh_server_init( &listen_path, &ca_file_paths, enforcement_mode, @@ -151,39 +168,22 @@ pub async fn run_ssh_server( } }; - let mut consecutive_resource_errors: u32 = 0; - let mut consecutive_unknown_errors: u32 = 0; - + let mut consecutive_resource_errors = 0; + let mut consecutive_unknown_errors = 0; loop { match listener.accept().await { Ok((stream, _peer)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; let config = config.clone(); - let policy = policy.clone(); - let workspace = workspace.clone(); - let proxy_url = proxy_url.clone(); - let ca_paths = ca_paths.clone(); - let provider_credentials = provider_credentials.clone(); - let user_environment = user_environment.clone(); - let main_session = Arc::clone(&main_session); + let port_forward = port_forward.clone(); + let boundary_exec = boundary_exec.clone(); + let main_session = main_session.clone(); tokio::spawn(async move { - if let Err(err) = handle_connection( - stream, - config, - policy, - workspace, - netns_fd, - proxy_url, - ca_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, - ) - .await + if let Err(err) = + handle_connection(stream, config, port_forward, boundary_exec, main_session) + .await { ocsf_emit!( SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -196,45 +196,31 @@ pub async fn run_ssh_server( } }); } - Err(err) => { - match classify_ssh_accept_error( - &err, - &mut consecutive_resource_errors, - &mut consecutive_unknown_errors, - ) { - SshAcceptAction::Terminal => { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message(format!( - "SSH accept loop exiting on terminal error: {err}" - )) - .build() - ); - break; - } - SshAcceptAction::Retry { backoff, severity } => { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(severity) - .status(StatusId::Failure) - .message(format!( - "SSH accept error (retrying in {}ms): {err}", - backoff.as_millis(), - )) - .build() - ); - tokio::time::sleep(backoff).await; - } + Err(error) => match classify_ssh_accept_error( + &error, + &mut consecutive_resource_errors, + &mut consecutive_unknown_errors, + ) { + SshAcceptAction::Terminal => { + return Err(error).into_diagnostic(); } - } + SshAcceptAction::Retry { backoff, severity } => { + ocsf_emit!( + SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(severity) + .status(StatusId::Failure) + .message(format!( + "SSH accept error (retrying in {}ms): {error}", + backoff.as_millis() + )) + .build() + ); + tokio::time::sleep(backoff).await; + } + }, } } - - Ok(()) } const MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS: u32 = 10; @@ -249,13 +235,13 @@ enum SshAcceptAction { } fn classify_ssh_accept_error( - err: &std::io::Error, + error: &std::io::Error, consecutive_resource_errors: &mut u32, consecutive_unknown_errors: &mut u32, ) -> SshAcceptAction { #[cfg(unix)] if matches!( - err.raw_os_error(), + error.raw_os_error(), Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK) ) { return SshAcceptAction::Terminal; @@ -263,7 +249,7 @@ fn classify_ssh_accept_error( #[cfg(unix)] if matches!( - err.raw_os_error(), + error.raw_os_error(), Some( libc::EMFILE | libc::ENFILE @@ -286,26 +272,20 @@ fn classify_ssh_accept_error( ) ) { *consecutive_unknown_errors = 0; - - #[cfg(unix)] - let is_resource_pressure = matches!( - err.raw_os_error(), + let resource_pressure = matches!( + error.raw_os_error(), Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM | libc::ENOSR) ); - #[cfg(not(unix))] - let is_resource_pressure = false; - - if is_resource_pressure { + if resource_pressure { *consecutive_resource_errors = consecutive_resource_errors.saturating_add(1); - let backoff_ms = 100u64 - .saturating_mul(1u64 << (*consecutive_resource_errors).min(7).saturating_sub(1)) + let backoff_ms = 100_u64 + .saturating_mul(1_u64 << (*consecutive_resource_errors).min(7).saturating_sub(1)) .min(5_000); return SshAcceptAction::Retry { backoff: Duration::from_millis(backoff_ms), severity: SeverityId::Medium, }; } - *consecutive_resource_errors = 0; return SshAcceptAction::Retry { backoff: Duration::from_millis(100), @@ -313,24 +293,25 @@ fn classify_ssh_accept_error( }; } - #[cfg(unix)] #[cfg(target_os = "linux")] - if matches!(err.raw_os_error(), Some(libc::ENONET)) { - *consecutive_unknown_errors = 0; + if error.raw_os_error() == Some(libc::ENONET) { *consecutive_resource_errors = 0; + *consecutive_unknown_errors = 0; return SshAcceptAction::Retry { backoff: Duration::from_millis(100), severity: SeverityId::Low, }; } + *consecutive_resource_errors = 0; *consecutive_unknown_errors = consecutive_unknown_errors.saturating_add(1); if *consecutive_unknown_errors >= MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS { - return SshAcceptAction::Terminal; - } - SshAcceptAction::Retry { - backoff: Duration::from_millis(100), - severity: SeverityId::Low, + SshAcceptAction::Terminal + } else { + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Low, + } } } @@ -338,16 +319,9 @@ fn classify_ssh_accept_error( async fn handle_connection( stream: tokio::net::UnixStream, config: Arc, - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Result<()> { // Access is gated by the Unix-socket filesystem permissions (root-only), // not by an application-level preface. The supervisor bridges the @@ -363,18 +337,7 @@ async fn handle_connection( .build() ); - let handler = SshHandler::new( - policy, - workspace, - netns_fd, - proxy_url, - ca_file_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, - ); + let handler = SshHandler::new(port_forward, boundary_exec, main_session); russh::server::run_stream(config, stream, handler) .await .map_err(|err| miette::miette!("ssh stream error: {err}"))?; @@ -387,13 +350,12 @@ async fn handle_connection( /// sender. This allows `window_change_request` to resize the correct PTY when /// multiple channels are open simultaneously (e.g. parallel shells, shell + /// sftp, etc.). -// Several independent per-channel boolean flags (login-shell opt-out and the -// main-attachment state bits) legitimately live side by side here. #[allow(clippy::struct_excessive_bools)] #[derive(Default)] struct ChannelState { input_sender: Option, - pty_master: Option, + process: Option>, + terminal: Option>, pty_request: Option, no_login_shell: bool, main_input_owner: Option, @@ -403,37 +365,6 @@ struct ChannelState { main_output_task: Option, } -const MAIN_DETACH_PREFIX: u8 = 0x10; // Ctrl-P -const MAIN_DETACH_KEY: u8 = 0x11; // Ctrl-Q - -/// Remove the `OpenShell` detach sequence from canonical-main input. -/// -/// A trailing Ctrl-P remains pending across SSH data frames. If the following -/// byte is not Ctrl-Q, both bytes are forwarded unchanged. Bytes after a -/// completed detach sequence are discarded because the attachment is closing. -fn filter_main_detach_sequence(prefix_pending: &mut bool, data: &[u8]) -> (Vec, bool) { - let mut forward = Vec::with_capacity(data.len() + usize::from(*prefix_pending)); - - for &byte in data { - if *prefix_pending { - if byte == MAIN_DETACH_KEY { - *prefix_pending = false; - return (forward, true); - } - forward.push(MAIN_DETACH_PREFIX); - *prefix_pending = false; - } - - if byte == MAIN_DETACH_PREFIX { - *prefix_pending = true; - } else { - forward.push(byte); - } - } - - (forward, false) -} - enum InputSender { Process(mpsc::Sender>), Main(tokio::sync::mpsc::Sender>), @@ -454,28 +385,26 @@ impl InputSender { } struct SshHandler { - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + /// Loopback port-forward, injected by the orchestrator (RFC 0012). In-pod + /// this connects from inside the workload netns; a delegated backend + /// tunnels into its guest. The handler does not know which. + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, channels: HashMap, } impl Drop for SshHandler { fn drop(&mut self) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; for state in self.channels.values_mut() { if state.main_attached { - self.main_session.end_terminal_attachment(); - state.main_attached = false; + main_session.end_terminal_attachment(); } if let Some(owner) = state.main_input_owner.take() { - self.main_session.release_input(owner); + main_session.release_input(owner); } if let Some(task) = state.main_output_task.take() { task.abort(); @@ -485,29 +414,14 @@ impl Drop for SshHandler { } impl SshHandler { - #[allow(clippy::too_many_arguments)] fn new( - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Self { Self { - policy, - workspace, - netns_fd, - proxy_url, - ca_file_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, + port_forward, + boundary_exec, main_session, channels: HashMap::new(), } @@ -550,15 +464,23 @@ impl russh::server::Handler for SshHandler { channel: ChannelId, _session: &mut Session, ) -> Result<(), Self::Error> { - if let Some(state) = self.channels.remove(&channel) { - if state.main_attached { - self.main_session.end_terminal_attachment(); - } - if let Some(owner) = state.main_input_owner { - self.main_session.release_input(owner); + if let Some(mut state) = self.channels.remove(&channel) { + if state.main_attached + && let Some(main_session) = self.main_session.as_ref() + { + main_session.end_terminal_attachment(); + if let Some(owner) = state.main_input_owner.take() { + main_session.release_input(owner); + } + if let Some(task) = state.main_output_task.take() { + task.abort(); + } + return Ok(()); } - if let Some(task) = state.main_output_task { - task.abort(); + if let Some(process) = state.process { + // Channel ownership defines the exec lifetime. Closing an SSH + // channel must not strand an in-boundary process. + let _ = process.terminate().await; } } Ok(()) @@ -574,12 +496,6 @@ impl russh::server::Handler for SshHandler { reply: ChannelOpenHandle, _session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - reply - .reject(ChannelOpenFailure::AdministrativelyProhibited) - .await; - return Ok(()); - } // Validate port range before truncating u32 -> u16. The SSH protocol // uses u32 for ports, but valid TCP ports are 0-65535. Without this // check, port 65537 truncates to port 1 (privileged). @@ -599,9 +515,8 @@ impl russh::server::Handler for SshHandler { return Ok(()); } - // Only allow forwarding to loopback destinations to prevent the - // sandbox SSH server from being used as a generic proxy. - if !is_loopback_host(host_to_connect) { + let target = direct_tcpip_target(host_to_connect, port_to_connect); + if target.is_none() { ocsf_emit!(SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Refuse) .action(ActionId::Denied) @@ -620,16 +535,13 @@ impl russh::server::Handler for SshHandler { let host = host_to_connect.to_string(); // SSH protocol port is bounded by u32 but only u16 is meaningful; // saturate as a guard for malformed clients. - let port = u16::try_from(port_to_connect).unwrap_or(u16::MAX); - let netns_fd = self.netns_fd; - - // Confirm the channel before spawning: the task below writes to it, and - // the peer must see the open-confirmation first. + let port = u16::try_from(port_to_connect).expect("port range checked above"); + let target = target.expect("loopback target checked above"); + let port_forward = self.port_forward.clone(); reply.accept().await; tokio::spawn(async move { - let addr = format!("{host}:{port}"); - let tcp = match connect_in_netns(&addr, netns_fd).await { + let mut tcp_stream = match port_forward.connect(target).await { Ok(stream) => stream, Err(err) => { ocsf_emit!( @@ -637,7 +549,9 @@ impl russh::server::Handler for SshHandler { .activity(ActivityId::Fail) .severity(SeverityId::Low) .status(StatusId::Failure) - .message(format!("direct-tcpip: failed to connect to {addr}: {err}")) + .message(format!( + "direct-tcpip: failed to connect to {host}:{port}: {err}" + )) .build() ); let _ = channel.close().await; @@ -646,7 +560,6 @@ impl russh::server::Handler for SshHandler { }; let mut channel_stream = channel.into_stream(); - let mut tcp_stream = tcp; let _ = tokio::io::copy_bidirectional(&mut channel_stream, &mut tcp_stream).await; }); @@ -694,18 +607,17 @@ impl russh::server::Handler for SshHandler { return Ok(()); }; if state.main_attached { - self.main_session - .resize(col_width, row_height, pixel_width, pixel_height); - } else if let Some(master) = state.pty_master.as_ref() { - let winsize = Winsize { - ws_row: to_u16(row_height.max(1)), - ws_col: to_u16(col_width.max(1)), - ws_xpixel: to_u16(pixel_width), - ws_ypixel: to_u16(pixel_height), - }; - if let Err(e) = unsafe_pty::set_winsize(master.as_raw_fd(), winsize) { - warn!("failed to resize PTY for channel {channel:?}: {e}"); + if let Some(main_session) = self.main_session.as_ref() { + main_session + .resize(col_width, row_height, pixel_width, pixel_height) + .await; } + } else if let Some(terminal) = state.terminal.as_ref() + && let Err(e) = terminal + .resize(to_u16(col_width.max(1)), to_u16(row_height.max(1))) + .await + { + warn!("failed to resize PTY for channel {channel:?}: {e}"); } Ok(()) } @@ -715,10 +627,6 @@ impl russh::server::Handler for SshHandler { channel: ChannelId, session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - session.channel_failure(channel)?; - return Ok(()); - } session.channel_success(channel)?; // Only allocate a PTY when the client explicitly requested one via // pty_request. VS Code Remote-SSH sends shell_request *without* a @@ -726,7 +634,7 @@ impl russh::server::Handler for SshHandler { // endings. Forcing a PTY here caused CRLF translation which made // VS Code misdetect the platform as Windows (and then try to run // `powershell`). - self.start_shell(channel, session.handle(), None)?; + self.start_shell(channel, session.handle(), None).await?; Ok(()) } @@ -736,16 +644,13 @@ impl russh::server::Handler for SshHandler { data: &[u8], session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - session.channel_failure(channel)?; - return Ok(()); - } session.channel_success(channel)?; let command = String::from_utf8_lossy(data).trim().to_string(); if command.is_empty() { return Ok(()); } - self.start_shell(channel, session.handle(), Some(command))?; + self.start_shell(channel, session.handle(), Some(command)) + .await?; Ok(()) } @@ -756,12 +661,11 @@ impl russh::server::Handler for SshHandler { session: &mut Session, ) -> Result<(), Self::Error> { if name == "openshell-main" { - if !self.channels.contains_key(&channel) { - return Err(anyhow::anyhow!( - "subsystem_request on unknown channel {channel:?}" - )); - } - if self.main_session.begin_terminal_attachment().is_err() { + let Some(main_session) = self.main_session.clone() else { + session.channel_failure(channel)?; + return Ok(()); + }; + if !begin_main_attachment(&main_session, self.channels.contains_key(&channel)) { session.channel_failure(channel)?; return Ok(()); } @@ -771,34 +675,33 @@ impl russh::server::Handler for SshHandler { .expect("main channel existence checked above"); state.main_attached = true; if let Some(pty) = state.pty_request.take() { - self.main_session.resize( - pty.col_width, - pty.row_height, - pty.pixel_width, - pty.pixel_height, - ); + main_session + .resize( + pty.col_width, + pty.row_height, + pty.pixel_width, + pty.pixel_height, + ) + .await; } - let (input, input_warning) = if state.main_read_only { + let (input, warning) = if state.main_read_only { (None, None) } else { - match self.main_session.acquire_input() { + match main_session.acquire_input() { Ok((owner, input)) => { state.main_input_owner = Some(owner); (Some(InputSender::Main(input)), None) } - Err(error) => { - warn!(%error, "main process input lease unavailable; attaching read-only"); - (None, Some(error)) - } + Err(error) => (None, Some(error)), } }; - state.main_detach_prefix_pending = false; state.input_sender = input; - let mut output = self.main_session.subscribe(); - let terminal_delivery = Arc::clone(&self.main_session); + state.main_detach_prefix_pending = false; + let mut output = main_session.subscribe(); + let terminal_delivery = main_session.clone(); let handle = session.handle(); session.channel_success(channel)?; - if let Some(error) = input_warning { + if let Some(error) = warning { let _ = handle .extended_data( channel, @@ -810,13 +713,13 @@ impl russh::server::Handler for SshHandler { let output_task = tokio::spawn(async move { loop { match output.recv().await { + Ok(MainOutput::Exit(code)) => { + terminal_delivery.wait_for_terminal_reported().await; + let _ = + send_main_output(&handle, channel, MainOutput::Exit(code)).await; + break; + } Ok(event) => { - if let MainOutput::Exit(code) = event { - terminal_delivery.wait_for_terminal_reported().await; - let _ = send_main_output(&handle, channel, MainOutput::Exit(code)) - .await; - break; - } let _ = send_main_output(&handle, channel, event).await; } Err(error) => { @@ -840,31 +743,24 @@ impl russh::server::Handler for SshHandler { if let Some(state) = self.channels.get_mut(&channel) { state.main_output_task = Some(output_task.abort_handle()); } - } else if name == "sftp" && !self.main_session.finished() { + } else if name == "sftp" { session.channel_success(channel)?; // sftp-server speaks the SFTP binary protocol over stdin/stdout, - // which is exactly what spawn_pipe_exec wires up. This enables + // which the boundary executor preserves as separate pipes. This enables // modern scp (SFTP-based, OpenSSH 9.0+) and SFTP clients to // transfer files into and out of the sandbox. - let input_sender = spawn_pipe_exec( - &self.policy, - &self.workspace, - Some("/usr/lib/openssh/sftp-server".to_string()), - false, - session.handle(), + self.start_exec_spec( channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &self.provider_credentials.child_env_with_gcp_resolved(), - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - let state = self.channels.get_mut(&channel).ok_or_else(|| { - anyhow::anyhow!("subsystem_request on unknown channel {channel:?}") - })?; - state.input_sender = Some(InputSender::Process(input_sender)); + session.handle(), + openshell_isolation_interface::contract::ExecSpec { + program: "/usr/lib/openssh/sftp-server".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }, + ) + .await?; } else { ocsf_emit!( SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -889,11 +785,9 @@ impl russh::server::Handler for SshHandler { ) -> Result<(), Self::Error> { // Accept the env request so the client knows we handled it, but we // don't actually propagate arbitrary variables — the sandbox - // environment is controlled via policy. We must reply so VSCode - // doesn't stall. Two exceptions carry supervisor signals the SSH - // protocol has no native field for: - // - OPENSHELL_NO_LOGIN_SHELL: gateway login-shell opt-out. - // - OPENSHELL_MAIN_READ_ONLY: read-only main attachment. + // environment is controlled via policy. The login-shell opt-out is a + // supervisor signal carried over SSH because the protocol has no + // native field for it. if variable_name == NO_LOGIN_SHELL_ENV.0 && let Some(state) = self.channels.get_mut(&channel) { @@ -919,38 +813,17 @@ impl russh::server::Handler for SshHandler { warn!("data on unknown channel {channel:?}"); return Ok(()); }; - - let main_attached = state.main_attached; - let (forward, detach) = if main_attached { + let (forward, detach) = if state.main_attached { filter_main_detach_sequence(&mut state.main_detach_prefix_pending, data) } else { (data.to_vec(), false) }; - let send_error = (!forward.is_empty()) + let error = (!forward.is_empty()) .then(|| state.input_sender.as_ref()?.send(forward).err()) .flatten(); - - if let Some(error) = send_error { - let handle = session.handle(); - if main_attached { - self.close_main_attachment(channel, handle, Some(error)) - .await; - } else { - let _ = handle - .extended_data( - channel, - 1, - format!("openshell: {error}; closing attachment\n").into_bytes(), - ) - .await; - let _ = handle.close(channel).await; - } - return Ok(()); - } - if detach { - self.close_main_attachment(channel, session.handle(), None) + if state.main_attached && (detach || error.is_some()) { + self.close_main_attachment(channel, session.handle(), error) .await; - return Ok(()); } Ok(()) } @@ -967,8 +840,9 @@ impl russh::server::Handler for SshHandler { if let Some(state) = self.channels.get_mut(&channel) { if state.main_attached && let Some(owner) = state.main_input_owner.take() + && let Some(main_session) = self.main_session.as_ref() { - self.main_session.release_input(owner); + main_session.release_input(owner); } state.input_sender.take(); state.main_detach_prefix_pending = false; @@ -984,47 +858,192 @@ impl russh::server::Handler for SshHandler { signal: Sig, _session: &mut Session, ) -> Result<(), Self::Error> { - if !self + if self .channels .get(&channel) .is_some_and(|state| state.main_attached) { + let signal = match signal { + Sig::HUP => Some(nix::sys::signal::Signal::SIGHUP), + Sig::INT => Some(nix::sys::signal::Signal::SIGINT), + Sig::KILL => Some(nix::sys::signal::Signal::SIGKILL), + Sig::QUIT => Some(nix::sys::signal::Signal::SIGQUIT), + Sig::TERM => Some(nix::sys::signal::Signal::SIGTERM), + _ => None, + }; + if let (Some(signal), Some(main_session)) = (signal, self.main_session.as_ref()) + && let Err(error) = main_session.signal_group(signal).await + { + warn!(%error, ?signal, "failed to signal canonical main process group"); + } return Ok(()); } + let Some(process) = self + .channels + .get(&channel) + .and_then(|state| state.process.clone()) + else { + return Ok(()); + }; let signal = match signal { - Sig::HUP => Some(nix::sys::signal::Signal::SIGHUP), - Sig::INT => Some(nix::sys::signal::Signal::SIGINT), - Sig::KILL => Some(nix::sys::signal::Signal::SIGKILL), - Sig::QUIT => Some(nix::sys::signal::Signal::SIGQUIT), - Sig::TERM => Some(nix::sys::signal::Signal::SIGTERM), + Sig::HUP => Some(openshell_isolation_interface::contract::BoundarySignal::Hup), + Sig::INT => Some(openshell_isolation_interface::contract::BoundarySignal::Int), + Sig::KILL => Some(openshell_isolation_interface::contract::BoundarySignal::Kill), + Sig::TERM => Some(openshell_isolation_interface::contract::BoundarySignal::Term), _ => None, }; if let Some(signal) = signal - && let Err(error) = self.main_session.signal_group(signal) + && let Err(error) = process.signal(signal).await { - warn!(%error, ?signal, "failed to signal canonical main process group"); + warn!(%error, ?signal, "failed to signal boundary exec process"); } Ok(()) } } -async fn send_main_output(handle: &Handle, channel: ChannelId, event: MainOutput) -> bool { - match event { - MainOutput::Stdout(data) => handle.data(channel, data).await.is_ok(), - MainOutput::Stderr(data) => handle.extended_data(channel, 1, data).await.is_ok(), - MainOutput::Exit(code) => { - let eof_sent = handle.eof(channel).await.is_ok(); - let status_sent = handle - .exit_status_request(channel, code.max(0).unsigned_abs()) +impl SshHandler { + async fn start_shell( + &mut self, + channel: ChannelId, + handle: Handle, + command: Option, + ) -> anyhow::Result<()> { + let state = self + .channels + .get_mut(&channel) + .ok_or_else(|| anyhow::anyhow!("start_shell on unknown channel {channel:?}"))?; + let no_login_shell = state.no_login_shell; + let pty = state.pty_request.take(); + let pty_requested = pty.is_some(); + let (program, args) = command.map_or_else( + || { + if pty_requested { + ("/bin/bash".to_string(), vec!["-i".to_string()]) + } else { + ("/bin/bash".to_string(), vec![]) + } + }, + |command| { + ( + "/bin/bash".to_string(), + vec![login_shell_flag(no_login_shell).to_string(), command], + ) + }, + ); + let env = pty + .as_ref() + .map(|request| vec![("TERM".to_string(), request.term.clone())]) + .unwrap_or_default(); + self.start_exec_spec( + channel, + handle, + openshell_isolation_interface::contract::ExecSpec { + program, + args, + env, + workdir: None, + pty: pty_requested, + }, + ) + .await?; + if let (Some(pty), Some(terminal)) = ( + pty, + self.channels + .get(&channel) + .and_then(|state| state.terminal.as_ref()), + ) { + terminal + .resize(to_u16(pty.col_width.max(1)), to_u16(pty.row_height.max(1))) .await - .is_ok(); - let close_sent = handle.close(channel).await.is_ok(); - eof_sent && status_sent && close_sent + .map_err(|error| anyhow::anyhow!(error.to_string()))?; } + Ok(()) + } + + async fn start_exec_spec( + &mut self, + channel: ChannelId, + handle: Handle, + spec: openshell_isolation_interface::contract::ExecSpec, + ) -> anyhow::Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let mut exec = self + .boundary_exec + .exec(spec) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; + let state = self + .channels + .get_mut(&channel) + .ok_or_else(|| anyhow::anyhow!("exec on unknown channel {channel:?}"))?; + state.process = Some(exec.process.clone()); + state.terminal = exec.terminal.take(); + + if let Some(mut stdin) = exec.stdin.take() { + let (sender, receiver) = mpsc::channel::>(); + let runtime = tokio::runtime::Handle::current(); + std::thread::spawn(move || { + while let Ok(bytes) = receiver.recv() { + if runtime.block_on(stdin.write_all(&bytes)).is_err() { + break; + } + } + }); + state.input_sender = Some(InputSender::Process(sender)); + } + + let mut stdout = exec.stdout; + let stdout_handle = handle.clone(); + let stdout_task = tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(size) => { + let _ = stdout_handle.data(channel, buffer[..size].to_vec()).await; + } + } + } + }); + let stderr_task = exec.stderr.map(|mut stderr| { + let stderr_handle = handle.clone(); + tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(size) => { + let _ = stderr_handle + .extended_data(channel, 1, buffer[..size].to_vec()) + .await; + } + } + } + }) + }); + tokio::spawn(async move { + let status = exec.process.wait().await; + let _ = stdout_task.await; + if let Some(task) = stderr_task { + let _ = task.await; + } + let code = match status { + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code)) => { + code.max(0).cast_unsigned() + } + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Signaled( + signal, + )) => (128_i32.saturating_add(signal)).max(0).cast_unsigned(), + Err(_) => 1, + }; + let _ = handle.eof(channel).await; + let _ = handle.exit_status_request(channel, code).await; + let _ = handle.close(channel).await; + }); + Ok(()) } -} -impl SshHandler { async fn close_main_attachment( &mut self, channel: ChannelId, @@ -1033,11 +1052,15 @@ impl SshHandler { ) { if let Some(state) = self.channels.get_mut(&channel) { if state.main_attached { - self.main_session.end_terminal_attachment(); + if let Some(main_session) = self.main_session.as_ref() { + main_session.end_terminal_attachment(); + } state.main_attached = false; } - if let Some(owner) = state.main_input_owner.take() { - self.main_session.release_input(owner); + if let Some(owner) = state.main_input_owner.take() + && let Some(main_session) = self.main_session.as_ref() + { + main_session.release_input(owner); } state.input_sender.take(); state.main_detach_prefix_pending = false; @@ -1058,62 +1081,25 @@ impl SshHandler { let _ = handle.exit_status_request(channel, 0).await; let _ = handle.close(channel).await; } +} - fn start_shell( - &mut self, - channel: ChannelId, - handle: Handle, - command: Option, - ) -> anyhow::Result<()> { - let provider_env = self.provider_credentials.child_env_with_gcp_resolved(); - let state = self - .channels - .get_mut(&channel) - .ok_or_else(|| anyhow::anyhow!("start_shell on unknown channel {channel:?}"))?; - let no_login_shell = state.no_login_shell; - if let Some(pty) = state.pty_request.take() { - // PTY was requested — allocate a real PTY (interactive shell or - // exec that explicitly asked for a terminal). - let (pty_master, input_sender) = spawn_pty_shell( - &self.policy, - &self.workspace, - command, - no_login_shell, - &pty, - handle, - channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &provider_env, - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - state.pty_master = Some(pty_master); - state.input_sender = Some(InputSender::Process(input_sender)); - } else { - // No PTY requested — use plain pipes so stdout/stderr are - // separate and output has clean LF line endings. This is the - // path VSCode Remote-SSH exec commands take. - let input_sender = spawn_pipe_exec( - &self.policy, - &self.workspace, - command, - no_login_shell, - handle, - channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &provider_env, - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - state.input_sender = Some(InputSender::Process(input_sender)); +fn begin_main_attachment(main_session: &MainSession, channel_exists: bool) -> bool { + channel_exists && main_session.begin_terminal_attachment().is_ok() +} + +async fn send_main_output(handle: &Handle, channel: ChannelId, event: MainOutput) -> bool { + match event { + MainOutput::Stdout(data) => handle.data(channel, data).await.is_ok(), + MainOutput::Stderr(data) => handle.extended_data(channel, 1, data).await.is_ok(), + MainOutput::Exit(code) => { + let eof = handle.eof(channel).await.is_ok(); + let status = handle + .exit_status_request(channel, code.max(0).unsigned_abs()) + .await + .is_ok(); + let close = handle.close(channel).await.is_ok(); + eof && status && close } - Ok(()) } } @@ -1131,12 +1117,11 @@ impl SshHandler { /// thread could be reused for unrelated tasks and must not be contaminated. /// On non-Linux platforms (no network namespace support), we connect directly. pub async fn connect_in_netns( - addr: &str, - netns_fd: Option, + addr: std::net::SocketAddr, + netns_fd: Option>, ) -> std::io::Result { #[cfg(target_os = "linux")] if let Some(fd) = netns_fd { - let addr = addr.to_string(); let (tx, rx) = tokio::sync::oneshot::channel(); std::thread::spawn(move || { let result = (|| -> std::io::Result { @@ -1144,11 +1129,11 @@ pub async fn connect_in_netns( // SAFETY: setns is safe to call; this is a dedicated thread that // will exit after the connection is established. #[allow(unsafe_code)] - let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; + let rc = unsafe { libc::setns(fd.as_raw_fd(), libc::CLONE_NEWNET) }; if rc != 0 { return Err(std::io::Error::last_os_error()); } - std::net::TcpStream::connect(&addr) + std::net::TcpStream::connect_timeout(&addr, Duration::from_secs(5)) })(); let _ = tx.send(result); }); @@ -1165,11 +1150,19 @@ pub async fn connect_in_netns( #[cfg(not(target_os = "linux"))] let _ = netns_fd; - let stream = tokio::net::TcpStream::connect(addr).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) + tokio::time::timeout( + Duration::from_secs(5), + connect_tcp_nodelay_best_effort(&[addr]), + ) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "connect timed out"))? } +const fn login_shell_flag(no_login_shell: bool) -> &'static str { + if no_login_shell { "-c" } else { "-lc" } +} + +#[allow(dead_code)] #[derive(Clone)] struct PtyRequest { term: String, @@ -1191,6 +1184,35 @@ impl Default for PtyRequest { } } +/// Derive the session USER and HOME from the policy's `run_as_user`. +/// +/// For name-based identities, looks up the home directory via `/etc/passwd` +/// (or defaults to `/home/{user}`). +/// +/// For numeric UIDs, there is no passwd entry — falls back to +/// `("{uid}", "/sandbox")` so the agent session still has a meaningful +/// USER identifier. +pub(crate) fn session_user_and_home(policy: &SandboxPolicy) -> (String, String) { + match policy.process.run_as_user.as_deref() { + Some(user) if !user.is_empty() => { + // Numeric UID — no passwd entry expected; use default HOME. + if user.parse::().is_ok() { + return (user.to_string(), "/sandbox".to_string()); + } + // Name-based identity — look up home from /etc/passwd. + let home = nix::unistd::User::from_name(user) + .ok() + .flatten() + .map_or_else( + || format!("/home/{user}"), + |u| u.dir.to_string_lossy().into_owned(), + ); + (user.to_string(), home) + } + _ => ("sandbox".to_string(), "/sandbox".to_string()), + } +} + #[allow(clippy::too_many_arguments)] pub(crate) fn apply_child_env( cmd: &mut Command, @@ -1238,16 +1260,12 @@ pub(crate) fn apply_child_env( } } -const fn login_shell_flag(no_login_shell: bool) -> &'static str { - if no_login_shell { "-c" } else { "-lc" } -} - #[allow(clippy::too_many_arguments)] +#[allow(dead_code)] fn spawn_pty_shell( policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, + workdir: Option, command: Option, - no_login_shell: bool, pty: &PtyRequest, handle: Handle, channel: ChannelId, @@ -1284,7 +1302,7 @@ fn spawn_pty_shell( }, |command| { let mut c = Command::new("/bin/bash"); - c.arg(login_shell_flag(no_login_shell)).arg(command); + c.arg("-lc").arg(command); c }, ); @@ -1297,7 +1315,7 @@ fn spawn_pty_shell( // Derive USER and HOME from the policy's run_as_user when available, // falling back to "sandbox" / "/sandbox" for backward compatibility. - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); + let (session_user, session_home) = session_user_and_home(policy); apply_child_env( &mut cmd, &session_home, @@ -1310,37 +1328,47 @@ fn spawn_pty_shell( ); cmd.stdin(stdin).stdout(stdout).stderr(stderr); - if let Some(dir) = workspace.root() { + if let Some(dir) = workdir.as_deref() { cmd.current_dir(dir); } // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); } // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] - let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workspace.root(), enforcement_mode) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; + let prepared_sandbox = { + let runtime_read_only = + crate::process::ca_runtime_read_only_paths(ca_file_paths.as_deref()); + crate::process::prepare_child_sandbox( + policy, + workdir.as_deref(), + enforcement_mode, + &runtime_read_only, + ) + .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))? + }; #[cfg(unix)] { unsafe_pty::install_pre_exec( &mut cmd, policy.clone(), - workspace.owned_root(), + workdir.clone(), slave_fd, netns_fd, resolved_identity, enforcement_mode, #[cfg(target_os = "linux")] prepared_sandbox, - ); + )?; } + #[cfg(target_os = "linux")] + let mut child_registry = managed_children::lock(); #[cfg(target_os = "linux")] let mut child = crate::process::spawn_std_command_with_supervisor_identity_namespace(cmd)?; #[cfg(not(target_os = "linux"))] @@ -1348,7 +1376,9 @@ fn spawn_pty_shell( #[cfg(target_os = "linux")] let child_pid = child.id(); #[cfg(target_os = "linux")] - let managed_child = managed_children::register(child_pid); + let managed_child = child_registry.register(child_pid); + #[cfg(target_os = "linux")] + drop(child_registry); let master_file = master; let (sender, receiver) = mpsc::channel::>(); @@ -1394,8 +1424,8 @@ fn spawn_pty_shell( std::thread::spawn(move || { let status = child.wait().ok(); #[cfg(target_os = "linux")] - if let Some(child) = managed_child { - managed_children::unregister(child); + if let Some(managed_child) = managed_child { + managed_children::unregister(managed_child); } let code = status.and_then(|s| s.code()).unwrap_or(1).unsigned_abs(); // Wait for the reader thread to finish forwarding all output before @@ -1422,11 +1452,11 @@ fn spawn_pty_shell( /// (type 1), preserving the separation that clients like `VSCode` Remote-SSH /// expect. Output retains clean LF line endings (no CRLF translation). #[allow(clippy::too_many_arguments)] +#[allow(dead_code)] fn spawn_pipe_exec( policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, + workdir: Option, command: Option, - no_login_shell: bool, handle: Handle, channel: ChannelId, netns_fd: Option, @@ -1449,15 +1479,15 @@ fn spawn_pipe_exec( }, |command| { let mut c = Command::new("/bin/bash"); - // Login shell (-l) sources .profile/.bashrc so tool env vars - // (VIRTUAL_ENV, etc.) are available. Callers that need a predictable - // environment opt out via OPENSHELL_NO_LOGIN_SHELL → plain -c. - c.arg(login_shell_flag(no_login_shell)).arg(command); + // Use login shell (-l) so that .profile/.bashrc are sourced and + // tool-specific env vars (VIRTUAL_ENV, UV_PYTHON_INSTALL_DIR, etc.) + // are available without hardcoding them here. + c.arg("-lc").arg(command); c }, ); - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); + let (session_user, session_home) = session_user_and_home(policy); apply_child_env( &mut cmd, &session_home, @@ -1472,36 +1502,46 @@ fn spawn_pipe_exec( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - if let Some(dir) = workspace.root() { + if let Some(dir) = workdir.as_deref() { cmd.current_dir(dir); } // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); } // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] - let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workspace.root(), enforcement_mode) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; + let prepared_sandbox = { + let runtime_read_only = + crate::process::ca_runtime_read_only_paths(ca_file_paths.as_deref()); + crate::process::prepare_child_sandbox( + policy, + workdir.as_deref(), + enforcement_mode, + &runtime_read_only, + ) + .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))? + }; #[cfg(unix)] { unsafe_pty::install_pre_exec_no_pty( &mut cmd, policy.clone(), - workspace.owned_root(), + workdir.clone(), netns_fd, resolved_identity, enforcement_mode, #[cfg(target_os = "linux")] prepared_sandbox, - ); + )?; } + #[cfg(target_os = "linux")] + let mut child_registry = managed_children::lock(); #[cfg(target_os = "linux")] let mut child = crate::process::spawn_std_command_with_supervisor_identity_namespace(cmd)?; #[cfg(not(target_os = "linux"))] @@ -1509,7 +1549,9 @@ fn spawn_pipe_exec( #[cfg(target_os = "linux")] let child_pid = child.id(); #[cfg(target_os = "linux")] - let managed_child = managed_children::register(child_pid); + let managed_child = child_registry.register(child_pid); + #[cfg(target_os = "linux")] + drop(child_registry); let child_stdin = child.stdin.take(); let child_stdout = child.stdout.take().expect("stdout must be piped"); @@ -1581,8 +1623,8 @@ fn spawn_pipe_exec( std::thread::spawn(move || { let status = child.wait().ok(); #[cfg(target_os = "linux")] - if let Some(child) = managed_child { - managed_children::unregister(child); + if let Some(managed_child) = managed_child { + managed_children::unregister(managed_child); } let code = status.and_then(|s| s.code()).unwrap_or(1).unsigned_abs(); // Wait for both reader threads. @@ -1648,12 +1690,9 @@ pub(crate) mod unsafe_pty { #[allow(unsafe_code)] #[allow(clippy::too_many_arguments)] - #[cfg_attr( - not(target_os = "linux"), - allow( - clippy::unnecessary_wraps, - reason = "Linux pre_exec setup can fail while non-Linux setup cannot." - ) + #[allow( + clippy::unnecessary_wraps, + reason = "keeps pre-exec installation fallible for callers if setup gains validation" )] pub fn install_pre_exec( cmd: &mut Command, @@ -1664,7 +1703,7 @@ pub(crate) mod unsafe_pty { resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, #[cfg(target_os = "linux")] prepared: Option, - ) { + ) -> anyhow::Result<()> { // Wrap in Option so we can .take() it out of the FnMut closure. // pre_exec is only called once (after fork, before exec). #[cfg(target_os = "linux")] @@ -1684,18 +1723,16 @@ pub(crate) mod unsafe_pty { ) }); } + Ok(()) } /// Pre-exec hook for pipe-based (non-PTY) exec. /// /// Skips `setsid` and `TIOCSCTTY` since there is no controlling terminal. #[allow(unsafe_code)] - #[cfg_attr( - not(target_os = "linux"), - allow( - clippy::unnecessary_wraps, - reason = "Linux pre_exec setup can fail while non-Linux setup cannot." - ) + #[allow( + clippy::unnecessary_wraps, + reason = "keeps pre-exec installation fallible for callers if setup gains validation" )] pub fn install_pre_exec_no_pty( cmd: &mut Command, @@ -1705,11 +1742,14 @@ pub(crate) mod unsafe_pty { resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, #[cfg(target_os = "linux")] prepared: Option, - ) { + ) -> anyhow::Result<()> { #[cfg(target_os = "linux")] let mut prepared = prepared; unsafe { cmd.pre_exec(move || { + if libc::setpgid(0, 0) != 0 { + return Err(std::io::Error::last_os_error()); + } enter_netns_and_sandbox( netns_fd, &policy, @@ -1720,6 +1760,7 @@ pub(crate) mod unsafe_pty { ) }); } + Ok(()) } fn enter_netns_and_sandbox( @@ -1810,6 +1851,34 @@ fn is_loopback_host(host: &str) -> bool { } } +/// Resolve a (loopback-validated) destination host string to an `IpAddr`, +/// mapping `localhost` to `127.0.0.1`. +/// +/// Returns `None` for anything that does not parse to an IP, so +/// [`LoopbackTarget::new`] never sees a hostname. +fn loopback_ip(host: &str) -> Option { + let host = host + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(host); + if host.eq_ignore_ascii_case("localhost") { + return Some(std::net::Ipv4Addr::LOCALHOST.into()); + } + host.parse().ok() +} + +fn direct_tcpip_target( + host: &str, + port: u32, +) -> Option { + if !is_loopback_host(host) { + return None; + } + let port = u16::try_from(port).ok()?; + let ip = loopback_ip(host)?; + openshell_isolation_interface::contract::LoopbackTarget::new(ip, port).ok() +} + #[cfg(test)] #[allow( clippy::doc_markdown, @@ -1820,18 +1889,221 @@ mod tests { use super::*; use std::process::Stdio; - /// Regression test: the direct-tcpip connect path sets `TCP_NODELAY`. + struct AcceptAnyServerKey; + + impl russh::client::Handler for AcceptAnyServerKey { + type Error = russh::Error; + + async fn check_server_key( + &mut self, + _server_public_key: &russh::keys::PublicKey, + ) -> Result { + Ok(true) + } + } + + struct TestPortForward; + + #[async_trait::async_trait] + impl openshell_isolation_interface::contract::BoundaryPortForward for TestPortForward { + async fn connect( + &self, + target: openshell_isolation_interface::contract::LoopbackTarget, + ) -> std::result::Result< + openshell_isolation_interface::contract::BoundaryDuplexStream, + openshell_isolation_interface::contract::BackendError, + > { + let stream = tokio::net::TcpStream::connect((target.host(), target.port())) + .await + .map_err(|error| { + openshell_isolation_interface::contract::BackendError::Process( + error.to_string(), + ) + })?; + Ok(Box::new(stream)) + } + } + + struct RejectingExec; + + #[async_trait::async_trait] + impl openshell_isolation_interface::contract::BoundaryExec for RejectingExec { + async fn exec( + &self, + _spec: openshell_isolation_interface::contract::ExecSpec, + ) -> std::result::Result< + openshell_isolation_interface::contract::ExecSession, + openshell_isolation_interface::contract::BackendError, + > { + Err( + openshell_isolation_interface::contract::BackendError::Unsupported( + "exec is not used by direct-tcpip tests".into(), + ), + ) + } + } + + async fn authenticated_test_client() -> russh::client::Handle { + let host_key = { + let mut rng = rand::rng(); + PrivateKey::random(&mut rng, Algorithm::Ed25519).expect("host key") + }; + let mut server_config = russh::server::Config { + auth_rejection_time: Duration::from_millis(1), + ..Default::default() + }; + server_config.keys.push(host_key); + + let handler = SshHandler::new( + Arc::new(TestPortForward), + Arc::new(RejectingExec), + Some(MainSession::inert()), + ); + let (server_stream, client_stream) = tokio::io::duplex(64 * 1024); + tokio::spawn(async move { + if let Ok(session) = + russh::server::run_stream(Arc::new(server_config), server_stream, handler).await + { + let _ = session.await; + } + }); + + let mut client = russh::client::connect_stream( + Arc::new(russh::client::Config::default()), + client_stream, + AcceptAnyServerKey, + ) + .await + .expect("SSH handshake should complete over the duplex"); + let auth = client + .authenticate_none("sandbox") + .await + .expect("auth_none should not error"); + assert!(matches!(auth, russh::client::AuthResult::Success)); + client + } + + #[cfg(unix)] + #[test] + fn transient_accept_errors_retry_with_bounded_backoff() { + let mut resource_errors = 0; + let mut unknown_errors = 0; + let aborted = std::io::Error::from_raw_os_error(libc::ECONNABORTED); + assert_eq!( + classify_ssh_accept_error(&aborted, &mut resource_errors, &mut unknown_errors), + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Low, + } + ); + + let exhausted = std::io::Error::from_raw_os_error(libc::EMFILE); + let first = + classify_ssh_accept_error(&exhausted, &mut resource_errors, &mut unknown_errors); + let second = + classify_ssh_accept_error(&exhausted, &mut resource_errors, &mut unknown_errors); + assert_eq!( + first, + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Medium, + } + ); + assert_eq!( + second, + SshAcceptAction::Retry { + backoff: Duration::from_millis(200), + severity: SeverityId::Medium, + } + ); + } + + #[cfg(unix)] + #[test] + fn invalid_listener_accept_error_is_terminal() { + let mut resource_errors = 0; + let mut unknown_errors = 0; + let error = std::io::Error::from_raw_os_error(libc::EBADF); + assert_eq!( + classify_ssh_accept_error(&error, &mut resource_errors, &mut unknown_errors), + SshAcceptAction::Terminal + ); + } + + #[test] + fn direct_tcpip_target_rejects_non_loopback_and_out_of_range_ports() { + assert!(direct_tcpip_target("10.0.0.1", 80).is_none()); + assert!(direct_tcpip_target("127.0.0.1", 65_537).is_none()); + } + + #[test] + fn direct_tcpip_target_accepts_loopback_destinations() { + let target = direct_tcpip_target("localhost", 8_080).expect("loopback target"); + assert_eq!( + target.host(), + std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST) + ); + assert_eq!(target.port(), 8_080); + } + #[tokio::test] - async fn connect_in_netns_sets_tcp_nodelay() { + async fn direct_tcpip_handler_rejects_invalid_destinations() { + for (host, port) in [("10.0.0.1", 80), ("127.0.0.1", 65_537)] { + let client = authenticated_test_client().await; + let error = client + .channel_open_direct_tcpip(host, port, "127.0.0.1", 0) + .await + .expect_err("invalid forwarding destination must be refused"); + assert!(matches!( + error, + russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) + )); + } + } + + #[tokio::test] + async fn direct_tcpip_handler_relays_loopback_bytes() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await - .expect("bind listener"); - let addr = listener.local_addr().expect("local addr"); + .expect("bind loopback echo listener"); + let port = listener.local_addr().expect("listener address").port(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept forwarded stream"); + let mut payload = [0_u8; 4]; + socket.read_exact(&mut payload).await.expect("read payload"); + socket.write_all(&payload).await.expect("echo payload"); + }); - let stream = connect_in_netns(&addr.to_string(), None) + let client = authenticated_test_client().await; + let channel = client + .channel_open_direct_tcpip("127.0.0.1", u32::from(port), "127.0.0.1", 0) .await - .expect("connect"); - assert!(stream.nodelay().expect("query TCP_NODELAY")); + .expect("loopback forwarding must be allowed"); + let mut stream = channel.into_stream(); + stream.write_all(b"ping").await.expect("write channel"); + let mut echoed = [0_u8; 4]; + tokio::time::timeout(Duration::from_secs(5), stream.read_exact(&mut echoed)) + .await + .expect("forwarded response timeout") + .expect("read channel"); + assert_eq!(&echoed, b"ping"); + } + + #[tokio::test] + async fn main_attachment_accepts_declared_session_after_process_exit() { + let main_session = MainSession::inert(); + assert!(main_session.finish(23, true).await); + assert!(main_session.finished()); + + assert!(begin_main_attachment(&main_session, true)); + let mut output = main_session.subscribe(); + assert!(matches!( + output.recv().await.expect("retained terminal status"), + MainOutput::Exit(23) + )); + main_session.end_terminal_attachment(); } #[cfg(unix)] @@ -1950,38 +2222,6 @@ mod tests { assert_eq!(output.stdout, b"hello"); } - /// Command execution selects a login shell by default and a non-login shell - /// under `--no-login-shell`, so user startup files are sourced only in the - /// default case. - #[cfg(unix)] - #[test] - fn login_shell_flag_controls_profile_sourcing() { - let home = tempfile::tempdir().unwrap(); - std::fs::write(home.path().join(".bash_profile"), "echo LOGIN_MARKER\n").unwrap(); - - let run = |flag: &str| -> String { - let out = Command::new("bash") - .arg(flag) - .arg("true") - .env("HOME", home.path()) - .env_remove("BASH_ENV") // isolate: -c still reads BASH_ENV if set - .output() - .expect("spawn bash"); - String::from_utf8_lossy(&out.stdout).into_owned() - }; - - assert_eq!(login_shell_flag(true), "-c"); - assert_eq!(login_shell_flag(false), "-lc"); - assert!( - run("-lc").contains("LOGIN_MARKER"), - "login shell must source .bash_profile" - ); - assert!( - !run("-c").contains("LOGIN_MARKER"), - "non-login shell must not source it" - ); - } - /// Verify that the stdin writer delivers all buffered data before exiting /// when the sender is dropped. This ensures channel_eof doesn't cause /// data loss — only signals "no more data after this". @@ -2191,56 +2431,6 @@ mod tests { assert_eq!(rx_b.recv().unwrap(), b"still-alive"); } - #[test] - fn main_detach_filter_forwards_ctrl_c_unchanged() { - let mut prefix_pending = false; - let (forward, detach) = - filter_main_detach_sequence(&mut prefix_pending, b"before\x03after"); - - assert_eq!(forward, b"before\x03after"); - assert!(!detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_removes_sequence_and_trailing_input() { - let mut prefix_pending = false; - let (forward, detach) = - filter_main_detach_sequence(&mut prefix_pending, b"before\x10\x11after"); - - assert_eq!(forward, b"before"); - assert!(detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_recognizes_sequence_across_frames() { - let mut prefix_pending = false; - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"before\x10"); - assert_eq!(forward, b"before"); - assert!(!detach); - assert!(prefix_pending); - - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"\x11"); - assert!(forward.is_empty()); - assert!(detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_forwards_unmatched_prefix() { - let mut prefix_pending = false; - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"\x10"); - assert!(forward.is_empty()); - assert!(!detach); - assert!(prefix_pending); - - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"x"); - assert_eq!(forward, b"\x10x"); - assert!(!detach); - assert!(!prefix_pending); - } - // ----------------------------------------------------------------------- // session_user_and_home tests (Phase 2: numeric UID support) // ----------------------------------------------------------------------- @@ -2260,33 +2450,12 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "1000"); // Numeric UID has no passwd entry — defaults to /sandbox. assert_eq!(home, "/sandbox"); } - #[test] - fn session_user_and_home_uses_driver_workspace_when_supplied() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("1234".into()), - run_as_group: Some("1235".into()), - }, - }; - - let (user, home) = session_user_and_home(&policy, Some("/workspace/project")); - assert_eq!(user, "1234"); - assert_eq!(home, "/workspace/project"); - } - #[test] fn session_user_and_home_returns_name_from_passwd() { use openshell_core::policy::{ @@ -2302,7 +2471,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "sandbox"); // Name-based — should resolve via passwd (or /home/{user}). assert!(!home.is_empty()); @@ -2323,7 +2492,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "sandbox"); assert_eq!(home, "/sandbox"); } @@ -2343,7 +2512,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "sandbox"); assert_eq!(home, "/sandbox"); } @@ -2363,7 +2532,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "1000660000"); assert_eq!(home, "/sandbox"); } @@ -2427,7 +2596,8 @@ mod tests { ) .expect("prepare should succeed in test environment"), ), - ); + ) + .expect("install pre_exec should succeed"); let output = cmd .spawn() @@ -2482,7 +2652,8 @@ mod tests { ProcessEnforcementMode::Full, #[cfg(target_os = "linux")] None, - ); + ) + .expect("install pre_exec should succeed"); let output = cmd .spawn() @@ -2495,299 +2666,4 @@ mod tests { "resolved-identity-ok" ); } - - // ----------------------------------------------------------------------- - // direct-tcpip authorization wiring (SEC-007) - // - // The `loopback_host_*` tests above cover the predicate in isolation. - // These drive the real `russh::server::Handler` over an in-memory duplex - // so the deny path itself is covered: channel-open authorization travels - // through a reply handle rather than the handler's return value, so a - // handler that never rejects anything still type-checks and still passes - // every predicate test. - // ----------------------------------------------------------------------- - - struct AcceptAnyServerKey; - - impl russh::client::Handler for AcceptAnyServerKey { - type Error = russh::Error; - - async fn check_server_key( - &mut self, - _server_public_key: &russh::keys::PublicKey, - ) -> Result { - Ok(true) - } - } - - fn forwarding_test_policy() -> SandboxPolicy { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - - SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - } - } - - /// Serve `SshHandler` on one end of an in-memory duplex and return an - /// authenticated client handle for the other end. - /// - /// The handler gets `netns_fd: None` so `connect_in_netns` performs a plain - /// TCP connect, making the forwarding path reachable without a network - /// namespace. - async fn authenticated_test_client_with_main( - main_session: Arc, - ) -> russh::client::Handle { - // Scoped so the `!Send` ThreadRng is dropped before the first await. - let host_key = { - let mut rng = rand::rng(); - PrivateKey::random(&mut rng, Algorithm::Ed25519).expect("host key") - }; - let mut server_config = russh::server::Config { - auth_rejection_time: Duration::from_millis(1), - ..Default::default() - }; - server_config.keys.push(host_key); - - let handler = SshHandler::new( - forwarding_test_policy(), - ResolvedWorkspace::default(), - None, - None, - None, - ProviderCredentialState::from_child_env_snapshot(0, HashMap::new()), - HashMap::new(), - ResolvedProcessIdentity::default(), - ProcessEnforcementMode::NetworkOnly, - main_session, - ); - - let (server_stream, client_stream) = tokio::io::duplex(64 * 1024); - tokio::spawn(async move { - if let Ok(session) = - russh::server::run_stream(Arc::new(server_config), server_stream, handler).await - { - let _ = session.await; - } - }); - - let mut client = russh::client::connect_stream( - Arc::new(russh::client::Config::default()), - client_stream, - AcceptAnyServerKey, - ) - .await - .expect("SSH handshake should complete over the duplex"); - - let auth = client - .authenticate_none("sandbox") - .await - .expect("auth_none should not error"); - assert!( - matches!(auth, russh::client::AuthResult::Success), - "sandbox SSH server accepts the none auth method" - ); - - client - } - - async fn authenticated_test_client() -> russh::client::Handle { - authenticated_test_client_with_main(MainSession::inert()).await - } - - #[tokio::test] - async fn abrupt_transport_drop_releases_main_input_lease() { - let main_session = MainSession::inert(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let channel = client.channel_open_session().await.expect("open session"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - match main_session.acquire_input() { - Err(_) => break, - Ok((owner, _)) => main_session.release_input(owner), - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should acquire canonical input lease"); - - drop(channel); - drop(client); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if main_session.acquire_input().is_ok() { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("handler drop should release canonical input lease"); - } - - #[tokio::test] - async fn main_attachment_closes_naturally_after_terminal_delivery() { - let main_session = MainSession::inert(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let mut channel = client.channel_open_session().await.expect("open session"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - match main_session.acquire_input() { - Err(_) => break, - Ok((owner, _)) => main_session.release_input(owner), - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should register its attachment"); - - assert!(main_session.finish(7, false).await); - main_session.mark_terminal_reported(); - - let exit_status = tokio::time::timeout(Duration::from_secs(1), async { - let mut exit_status = None; - loop { - match channel.wait().await { - Some(russh::ChannelMsg::ExitStatus { - exit_status: status, - }) => { - exit_status = Some(status); - } - Some(russh::ChannelMsg::Close) => break exit_status, - None => panic!("main channel ended without a close message"), - Some(_) => {} - } - } - }) - .await - .expect("main channel should deliver its exit status"); - assert_eq!(exit_status, Some(7)); - drop(channel); - drop(client); - - tokio::time::timeout( - Duration::from_secs(1), - main_session.wait_for_terminal_attachments(), - ) - .await - .expect("peer channel close should release terminal delivery"); - } - - #[tokio::test] - async fn main_subsystem_applies_initial_pty_dimensions() { - let (main_session, _slave) = MainSession::terminal_for_test(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let channel = client.channel_open_session().await.expect("open session"); - channel - .request_pty(true, "xterm-256color", 200, 60, 1600, 900, &[]) - .await - .expect("request PTY"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if main_session.terminal_size_for_test() == (200, 60) { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should apply the initial PTY dimensions"); - } - - #[tokio::test] - async fn direct_tcpip_rejects_non_loopback_destination() { - let client = authenticated_test_client().await; - - let err = client - .channel_open_direct_tcpip("10.0.0.1", 80, "127.0.0.1", 0) - .await - .expect_err("forwarding to a non-loopback host must be refused"); - - assert!( - matches!( - err, - russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) - ), - "expected AdministrativelyProhibited, got {err:?}" - ); - } - - #[tokio::test] - async fn direct_tcpip_rejects_port_above_tcp_range() { - let client = authenticated_test_client().await; - - // 65_537 truncates to port 1 when cast to u16, so the guard has to - // reject it before the cast rather than forward to a privileged port. - let err = client - .channel_open_direct_tcpip("127.0.0.1", 65_537, "127.0.0.1", 0) - .await - .expect_err("a port outside the TCP range must be refused"); - - assert!( - matches!( - err, - russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) - ), - "expected AdministrativelyProhibited, got {err:?}" - ); - } - - #[tokio::test] - async fn direct_tcpip_forwards_to_loopback_listener() { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind loopback echo listener"); - let port = listener.local_addr().expect("listener address").port(); - tokio::spawn(async move { - if let Ok((mut socket, _)) = listener.accept().await { - let mut buf = [0u8; 64]; - if let Ok(n) = socket.read(&mut buf).await - && n > 0 - { - let _ = socket.write_all(&buf[..n]).await; - } - } - }); - - let client = authenticated_test_client().await; - let channel = client - .channel_open_direct_tcpip("127.0.0.1", u32::from(port), "127.0.0.1", 0) - .await - .expect("forwarding to a loopback listener must be allowed"); - - let mut stream = channel.into_stream(); - stream.write_all(b"ping").await.expect("write to channel"); - - let mut echoed = [0u8; 4]; - tokio::time::timeout(Duration::from_secs(10), stream.read_exact(&mut echoed)) - .await - .expect("relayed response should arrive before the timeout") - .expect("read from channel"); - assert_eq!(&echoed, b"ping", "bytes round-trip through the tunnel"); - } } diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 98a3c0497b..203c06d17b 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -11,8 +11,6 @@ //! selection — it has no protocol awareness of the bytes flowing through. use std::net::IpAddr; -#[cfg(target_os = "linux")] -use std::os::fd::RawFd; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; @@ -23,6 +21,7 @@ use openshell_core::proto::{ RelayOpenResult, ReportMainProcessExitRequest, SupervisorHeartbeat, SupervisorHello, SupervisorMessage, TcpRelayTarget, gateway_message, relay_open, supervisor_message, }; +use openshell_isolation_interface::contract::{BoundaryPortForward, LoopbackTarget}; use openshell_ocsf::{ ActivityId, ConnectionInfo, Endpoint, NetworkActivityBuilder, OcsfEvent, SandboxContext, SeverityId, StatusId, ocsf_emit, @@ -33,7 +32,6 @@ use tokio_stream::StreamExt; use tracing::{debug, warn}; use openshell_core::grpc_client; -use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::transport_errors::is_expected_transport_close_status; const INITIAL_BACKOFF: Duration = Duration::from_secs(1); @@ -278,31 +276,59 @@ pub fn spawn( endpoint: String, sandbox_id: String, ssh_socket_path: std::path::PathBuf, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, instance_id: String, ) -> tokio::task::JoinHandle<()> { + spawn_with_readiness( + endpoint, + sandbox_id, + ssh_socket_path, + port_forward, + expected_ssh_peer_pid, + terminating, + instance_id, + ) + .0 +} + +/// Spawn the supervisor session and expose when the gateway has accepted it. +pub fn spawn_with_readiness( + endpoint: String, + sandbox_id: String, + ssh_socket_path: std::path::PathBuf, + port_forward: Arc, + expected_ssh_peer_pid: Option, + terminating: Arc, + instance_id: String, +) -> ( + tokio::task::JoinHandle<()>, + tokio::sync::watch::Receiver, +) { + let (ready_tx, ready_rx) = tokio::sync::watch::channel(false); let config = SessionConfig { endpoint, sandbox_id, ssh_socket_path, - netns_fd, + port_forward, expected_ssh_peer_pid, terminating, instance_id, + ready_tx, }; - tokio::spawn(run_session_loop(config)) + (tokio::spawn(run_session_loop(config)), ready_rx) } struct SessionConfig { endpoint: String, sandbox_id: String, ssh_socket_path: std::path::PathBuf, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, instance_id: String, + ready_tx: tokio::sync::watch::Sender, } async fn run_session_loop(config: SessionConfig) { @@ -392,6 +418,8 @@ async fn run_single_session( heartbeat_secs, ); ocsf_emit!(event); + config.ready_tx.send_replace(true); + // Main loop: receive gateway messages + send heartbeats. let mut heartbeat_interval = tokio::time::interval(Duration::from_secs(u64::from(heartbeat_secs))); @@ -411,7 +439,7 @@ async fn run_single_session( let context = GatewayMessageContext { sandbox_id: &config.sandbox_id, ssh_socket_path: &config.ssh_socket_path, - netns_fd: config.netns_fd, + port_forward: &config.port_forward, expected_ssh_peer_pid: config.expected_ssh_peer_pid, channel: &channel, tx: &tx, @@ -479,7 +507,7 @@ pub async fn finalize_main_process_exit( struct GatewayMessageContext<'a> { sandbox_id: &'a str, ssh_socket_path: &'a std::path::Path, - netns_fd: Option, + port_forward: &'a Arc, expected_ssh_peer_pid: Option, channel: &'a grpc_client::AuthedChannel, tx: &'a mpsc::Sender, @@ -498,7 +526,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< let channel = context.channel.clone(); let ssh_socket_path = context.ssh_socket_path.to_path_buf(); let tx = context.tx.clone(); - let netns_fd = context.netns_fd; + let port_forward = context.port_forward.clone(); let expected_ssh_peer_pid = context.expected_ssh_peer_pid; let terminating = Arc::clone(context.terminating); @@ -510,7 +538,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< match handle_relay_open( relay_open, &ssh_socket_path, - netns_fd, + port_forward, expected_ssh_peer_pid, channel, tx, @@ -567,7 +595,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< async fn handle_relay_open( relay_open: RelayOpen, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, channel: grpc_client::AuthedChannel, tx: mpsc::Sender, @@ -577,7 +605,7 @@ async fn handle_relay_open( let target = match open_target( &relay_open, ssh_socket_path, - netns_fd, + &port_forward, expected_ssh_peer_pid, ) .await @@ -722,11 +750,11 @@ async fn send_relay_open_result( async fn open_target( relay_open: &RelayOpen, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: &Arc, expected_ssh_peer_pid: Option, ) -> Result, Box> { match relay_open.target.as_ref() { - Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, netns_fd).await, + Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, port_forward).await, Some(relay_open::Target::Ssh(_)) | None => { let runtime_path = crate::unix_socket::runtime_path(ssh_socket_path); let stream = tokio::net::UnixStream::connect(runtime_path.as_ref()).await?; @@ -747,59 +775,27 @@ async fn open_target( async fn open_tcp_target( target: &TcpRelayTarget, - netns_fd: Option, + port_forward: &Arc, ) -> Result, Box> { let host = normalize_tcp_target_host(target)?; let port = u16::try_from(target.port).map_err(|_| "tcp target port must fit in u16")?; - let stream = connect_tcp_target(host, port, netns_fd).await?; + // `normalize_tcp_target_host` returns a loopback IP string; parse it and let + // `LoopbackTarget::new` re-validate before connecting. + let ip: IpAddr = host + .parse() + .map_err(|_| "tcp target host must be a loopback IP")?; + let target = LoopbackTarget::new(ip, port) + .map_err(|e| -> Box { e.to_string().into() })?; + // Connect inside the boundary through the injected port-forward interface + // (RFC 0012). In-pod this enters the workload netns, a delegated backend + // tunnels into its guest. + let stream = port_forward + .connect(target) + .await + .map_err(|e| -> Box { e.to_string().into() })?; Ok(Box::new(stream)) } -#[cfg(target_os = "linux")] -async fn connect_tcp_target( - host: String, - port: u16, - netns_fd: Option, -) -> Result> { - if let Some(fd) = netns_fd { - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result { - #[allow(unsafe_code)] - let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - std::net::TcpStream::connect((host.as_str(), port)) - })(); - let _ = tx.send(result); - }); - - let stream = rx - .await - .map_err(|_| "netns tcp connect thread panicked")??; - stream.set_nonblocking(true)?; - let stream = tokio::net::TcpStream::from_std(stream)?; - set_tcp_nodelay_best_effort(&stream); - return Ok(stream); - } - - let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) -} - -#[cfg(not(target_os = "linux"))] -async fn connect_tcp_target( - host: String, - port: u16, - _netns_fd: Option, -) -> Result> { - let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) -} - #[cfg(test)] fn validate_tcp_target(target: &TcpRelayTarget) -> Result<(), String> { normalize_tcp_target_host(target).map(|_| ()) @@ -839,20 +835,6 @@ mod target_tests { } } - /// Regression test: the TCP relay connect path sets `TCP_NODELAY`. - #[tokio::test] - async fn connect_tcp_target_sets_tcp_nodelay() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind listener"); - let addr = listener.local_addr().expect("local addr"); - - let stream = connect_tcp_target(addr.ip().to_string(), addr.port(), None) - .await - .expect("connect"); - assert!(stream.nodelay().expect("query TCP_NODELAY")); - } - #[test] fn tcp_target_allows_loopback_hosts() { validate_tcp_target(&tcp("127.0.0.1", 8080)).expect("ipv4 loopback"); @@ -1135,7 +1117,12 @@ mod ocsf_event_tests { }); let relay = ssh_relay_open("peer-check"); - let trusted = open_target(&relay, &socket, None, Some(std::process::id())) + // The SSH relay path does not use the port-forward (that is the TCP + // target path); connect from the supervisor's own namespace. + let port_forward: Arc = + Arc::new(crate::boundary_io::NetnsPortForward::new(None, None)); + + let trusted = open_target(&relay, &socket, &port_forward, Some(std::process::id())) .await .expect("matching peer PID should be accepted"); drop(trusted); @@ -1143,7 +1130,7 @@ mod ocsf_event_tests { let Err(err) = open_target( &relay, &socket, - None, + &port_forward, Some(std::process::id().saturating_add(1)), ) .await diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index d515fd70b1..51512f3aa5 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -6,13 +6,23 @@ # The static sandbox binary is staged at: # deploy/docker/.build/prebuilt-binaries//openshell-sandbox # -# Alpine supplies nftables and iptables for pod-namespace egress enforcement. +# Alpine supplies the trusted helper runtime used by boundary mode for +# namespace setup and egress enforcement. The runtime is copied into workload +# images as a root-owned, read-only directory instead of trusting their tools. FROM alpine:3.22 AS supervisor ARG TARGETARCH -RUN apk add --no-cache nftables iptables iptables-legacy +RUN apk add --no-cache iproute2 nftables iptables iptables-legacy \ + && mkdir -p /openshell-runtime \ + && cp -aL /bin /sbin /lib /usr/bin /usr/sbin /usr/lib /openshell-runtime/ \ + && mkdir -p /openshell-runtime/etc /openshell-runtime/usr/share \ + && if [ -d /etc/iproute2 ]; then cp -aL /etc/iproute2 /openshell-runtime/etc/; fi \ + && if [ -d /usr/share/nftables ]; then cp -aL /usr/share/nftables /openshell-runtime/usr/share/; fi \ + && test -x /openshell-runtime/bin/sh \ + && test -x /openshell-runtime/sbin/ip \ + && test -x /openshell-runtime/sbin/nft # Keep the binary root-owned for Podman image-volume mounts and executable by # the Kubernetes network sidecar's non-root proxy UID. diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md index 2be6b3f6d5..48e2ae3da1 100644 --- a/rfc/0012-isolation-backend/README.md +++ b/rfc/0012-isolation-backend/README.md @@ -19,7 +19,7 @@ links: Today the supervisor both builds the workload's isolation boundary and applies its network policy. Because the supervisor runs inside the agent container, the privilege needed to build that boundary sits beside the code it confines. This RFC moves boundary construction and process operations behind a pluggable **Isolation Backend**. The supervisor continues to apply approved network policy through network mediation. -The compute driver provisions the workload and trusted components. The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. The backend establishes the isolation controls, manages workload processes, and routes egress to network mediation. The same lifecycle supports today's in-pod implementation and future delegated implementations without topology-specific supervisor paths. +The compute driver prepares the workload topology and trusted inputs. The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. OpenShell packages that role as `openshell-sandbox --mode=control`. When the workload is separated by a container, pod, userspace kernel, or VM boundary, the same binary runs a small trusted counterpart as `openshell-sandbox --mode=boundary`. The boundary mode owns process observation and operations that cannot be implemented portably from outside the boundary; it has no gateway credentials and no policy authority. ## Motivation @@ -33,56 +33,78 @@ All three come from coupling boundary construction to boundary operation. A comm ## Non-goals -- **Implementing a delegated backend.** Each topology requires its own design and implementation. +- **Standardizing a topology's resource API.** Docker, Kubernetes, VM, and other resource mechanisms remain backend-specific. - **Changing authorization.** [RFC 0001](../0001-core-architecture/README.md) owns control-plane and sandbox identity. A delegated backend must still authenticate callers and scope them to one boundary. -- **Standardizing backend-internal component coordination.** A backend may coordinate helper, sidecar, or interception processes behind one lifecycle; how those components cooperate is backend-specific, not contract surface. +- **Standardizing resource-specific provisioning or transport setup.** A driver still decides how to place the binary, create a private Unix socket, authenticated TCP endpoint, or vsock endpoint, and establish kernel-specific egress capture. This RFC standardizes the authenticated control-to-boundary messages carried over that endpoint. - **Changing gateway lifecycle or public status.** This RFC adds no gateway activation operation, public phase, or status API, and it does not define how a boundary's effective isolation model is surfaced to operators. ## Proposal The mental model has three roles: -- The **compute driver** provisions the sandbox instance according to the selected placement of the workload and trusted isolation components. That placement is the **topology**. +- The **compute driver** prepares placement and trusted topology inputs and owns durable resource provisioning, deletion, and reconciliation. That placement is the **topology**. - The **Isolation Backend** establishes and operates the topology-specific controls around the workload. It also routes workload egress to network mediation and provides process operations. - The **logical supervisor** is the trusted control-plane bridge between the gateway and the workload. It drives the backend, handles authorized gateway requests, and applies approved network policy through network mediation. Together, network policy, filesystem isolation, syscall filtering, and sandbox identity form the workload's isolation boundary. The roles above enforce that boundary and may run in one process or across several trusted components. Their placement does not change the contract. -Each active boundary has at most one logical supervisor, which may span multiple coupled processes. The backend routes all workload egress through a per-boundary source, and the supervisor consumes that source. Internal delegation and transport remain topology-private. +Each active boundary has exactly one control role and at most one boundary role. Together they implement one logical supervisor; boundary mode is not an independently authorized supervisor. The control role owns the gateway session, admitted policy, network-policy decisions, and RFC 0012 lifecycle. Boundary mode owns boundary-local process groups, `exec`, signal, wait, PTY, loopback forwarding, binary observation, and egress capture. The transport and physical placement remain driver-owned. [RFC 0001](../0001-core-architecture/README.md) continues to own sandbox authentication and authorization. In this contract, sandbox identity means binding the authenticated sandbox context to the isolation boundary. -Admission selects the sandbox's topology and determines its trusted context. The compute driver sets up the topology and gives the logical supervisor a `TopologyDescriptor` describing what it provisioned. The supervisor uses the descriptor to attach the matching Isolation Backend. The backend prepares the required controls before the agent starts. +Admission selects the sandbox's topology and trusted context. The compute driver gives the logical supervisor a `TopologyDescriptor` for the matching backend. Its opaque payload can identify an existing resource or carry trusted prepared inputs from which `attach` establishes the boundary. The backend prepares the required controls before the agent starts. ```mermaid flowchart TB Gateway["Gateway"] -->|"create sandbox"| Driver["Compute driver"] - subgraph Topology["Driver-provisioned topology (placement varies)"] - Supervisor["Supervisor"] - Backend["Isolation Backend (may coordinate components)"] + subgraph Topology["Admitted topology (placement varies)"] + Control["openshell-sandbox
--mode=control"] + Backend["Remote Isolation Backend"] subgraph Boundary["Isolation boundary"] - Mediator["Network mediation"] + BoundaryAgent["openshell-sandbox
--mode=boundary"] subgraph Execution["Workload execution environment"] Workload["Workload"] end end - - Supervisor -->|"drives contract"| Backend - Backend -->|"establishes and confirms"| Boundary - Backend -.->|"routes all workload egress to"| Mediator - Supervisor -.->|"applies network policy through"| Mediator - Backend -->|"after Ready: makes admitted agent runnable"| Workload - Workload ==>|"only egress"| Mediator + Mediator["Network mediation"] + + Control -->|"drives RFC 0012"| Backend + Backend <-->|"authenticated, versioned
boundary protocol"| BoundaryAgent + BoundaryAgent -->|"start / exec / signal / wait"| Workload + Workload ==>|"captured egress"| BoundaryAgent + BoundaryAgent ==>|"attributed streams"| Mediator + Control -.->|"policy decisions"| Mediator end - Driver -->|"resources + TopologyDescriptor"| Supervisor + Driver -->|"resources + protected configs"| BoundaryAgent + Driver -->|"trusted TopologyDescriptor"| Control + Gateway <-->|"authorized session"| Control Mediator -->|"allowed egress"| Egress["Egress"] ``` -In the in-pod topology, the supervisor drives a backend implemented in the same process. Other topologies may delegate backend operations without changing the supervisor lifecycle. +Co-located deployments may keep the existing in-process backend and omit boundary mode. Separated deployments use the shared remote backend and boundary protocol, so adding a driver changes provisioning and transport selection without adding a topology branch to the control role. + +### Supervisor modes and boundary protocol + +`openshell-sandbox --mode=control` runs outside the untrusted execution environment. It receives the admitted backend name and a protected `TopologyDescriptor`, resolves the backend without fallback, and owns the gateway-facing access plane. It is the only mode that possesses gateway credentials or applies approved network policy. + +Orchestrators may run control mode with `--health-check --health-port `. +The listener defaults to `0.0.0.0`; `--health-bind-ip ` (or +`OPENSHELL_HEALTH_BIND_IP`) selects an explicit IPv4 or IPv6 address. Kubernetes +drivers should populate the environment variable from the Downward API +`status.podIP`, which keeps the probe address family aligned with the pod. +The TCP listener becomes reachable only after `start_agent` has returned a +`RunningBoundary` and the gateway-facing access plane is established. Control +mode drops the listener when that boundary/access-plane lifetime ends, so a +Kubernetes `tcpSocket` readiness probe observes semantic control readiness +rather than mere process liveness. The listener carries no application data. + +`openshell-sandbox --mode=boundary --boundary-config ` runs inside, or immediately adjacent to, the execution environment. The driver supplies the config over a protected filesystem channel. It names one boundary, one private listener, one per-boundary bootstrap credential, and the workload identity. The identity is either a platform-resolved numeric UID/GID pair or an OCI `Config.User` declaration that boundary mode resolves against the workload filesystem. Boundary mode authenticates and scopes every request to that boundary. It never accepts policy or identity claims from workload code and it cannot authorize an operation independently of control mode. -A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver owns the sandbox instance and topology lifecycle. +The shared protocol is versioned independently of a driver's resource descriptor. It carries `attach`, `confirm`, `start_agent`, agent and exec `wait`/`signal`/`terminate`, PTY resize, loopback-only forwarding, and attributed egress streams. Drivers may use a private Unix socket, TLS-authenticated TCP, a Unix endpoint mapped to guest vsock, or host `AF_VSOCK`; adding a transport does not change lifecycle or operation semantics. A TCP transport that crosses a shared or operator-managed network must authenticate the server name against a driver-provisioned trust root and encrypt every control and stream request. Network isolation alone is not a confidentiality boundary. Secrets are delivered in protected files, redacted from debug output, and never placed in workload-visible environment or process arguments. When a bind-mounted file's host owner may match the workload UID, the driver requires boundary mode to re-own it as root-only before the first workload instruction. Unix socket inodes permit cross-UID control within their private driver-owned directory; the per-boundary bootstrap credential authenticates every request. + +A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver or external orchestrator owns the durable resource lifecycle even when the backend establishes a resource as part of `attach`. ### Contract invariants @@ -91,7 +113,7 @@ Six invariants hold for every boundary: 1. Workload egress is denied except through network mediation for the boundary's lifetime. 2. No untrusted instruction executes until every admitted control applicable to that process is in force. 3. An operation is authorized only when the complete effective policy permits it; network operations are decided through network mediation. There is no silent weakening. -4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the compute driver's provisioned execution environment. +4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the admitted execution environment. 5. Shared infrastructure preserves strict per-boundary lifecycle, policy, identity, enforcement, and cleanup isolation. 6. If the logical supervisor is lost, the boundary remains under its last confirmed enforcement state while supervisor-dependent operations fail closed. Loss of required enforcement ends `Running` and terminates all workload processes within a documented bound; detection and termination may be performed by a trusted node or control-plane actor. Network-mediation unavailability denies outbound connections and never enables direct egress. @@ -99,21 +121,22 @@ Each backend states its termination bound in its implementation documentation. L ### Provisioning -Provisioning runs on the control plane, and three rules hold in every topology: +Provisioning is selected by trusted control-plane configuration, and four rules hold in every topology: -1. **Admission selects the topology** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. The `TopologyDescriptor` supplied by the compute driver must name that backend, and resolution never falls back to another backend. -2. **The compute driver provisions the topology** and anything the selected backend needs. -3. **The backend establishes standing enforcement before untrusted code runs**, during provisioning or `attach`, depending on the backend. +1. **Admission selects the topology** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. The `TopologyDescriptor` must name that backend, and resolution never falls back to another backend. +2. **The descriptor supports prepared and existing resources.** A compute driver or orchestrator may identify an existing resource, or it may supply trusted prepared inputs that the backend uses to establish the resource during `attach`. +3. **The compute driver owns the durable resource lifecycle.** It provisions or prepares, deletes, and reconciles the topology independently of logical-supervisor availability. +4. **The backend establishes standing enforcement before untrusted code runs**, during `attach` or `confirm`, depending on the backend. If a topology depends on cluster-scoped coverage or registration, admission verifies that the prerequisite covers the boundary's placement before untrusted code runs. Every topology provides a trusted cleanup path that does not depend on logical-supervisor availability. -A compute driver may provision a resource and `TopologyDescriptor` before the control plane assigns it to a sandbox. No untrusted workload runs while the resource is unassigned. After claim or assignment produces a trusted `SandboxContext`, the supervisor calls `attach`; the backend either binds that context to the prepared resource and returns `Bound`, or rejects it as incompatible. Pool creation, claim, reset, release, and recycling remain outside this contract. +A compute driver may provision a resource and `TopologyDescriptor` before the control plane assigns it to a sandbox. No untrusted workload runs while the resource is unassigned. It may instead prepare immutable image or disk identities, normalized runtime settings, placement results, or protected references to artifacts and encode those inputs in the descriptor. After assignment produces a trusted `SandboxContext`, the supervisor calls `attach`; the backend atomically establishes or locates the resource, binds that context, and returns `Bound`, or rejects it as incompatible. Attach-time establishment is idempotent on trusted sandbox identity and launch generation. Partial resources are removed or remain labeled for compute-driver reconciliation. Pool creation, claim, reset, release, and recycling remain outside this contract. ### The topology descriptor -The driver supplies a descriptor for every topology admitted to this contract, including in-pod and resources prepared before assignment. The common envelope names the backend and carries an opaque payload. +The compute driver supplies a descriptor for every topology admitted to this contract. The common envelope names the backend and carries an opaque payload. ```rust struct TopologyDescriptor { @@ -125,7 +148,7 @@ struct TopologyDescriptor { `version` is the Isolation Backend interface version. Backend name and version match exactly; this contract does not negotiate compatibility ranges. The descriptor is transport-neutral. Provisioning supplies it to the supervisor before `attach`; how it is transported is topology-specific and outside this contract, and every transport preserves one property: workload-controlled input cannot select or modify the descriptor. -The opaque payload identifies, or gives the backend enough information to resolve, the exact driver-provisioned resource. It may also carry topology-specific endpoint or helper-role information; there are no common topology or role fields. +The opaque payload identifies an existing resource or gives the backend trusted prepared inputs with which to establish the exact resource during `attach`. It may also carry topology-specific endpoint or helper-role information; there are no common topology or role fields. Common verification requires: @@ -133,13 +156,13 @@ Common verification requires: - the descriptor's version is one the supervisor supports, and the resolved backend reports that same version; and - `SandboxContext` is constructed after the control plane assigns the resource to the admitted sandbox, using authenticated control-plane and trusted supervisor state. -The supervisor validates the descriptor's common fields and produces a `VerifiedTopologyDescriptor`, then resolves its `backend_name` and version without fallback. Verification does not imply that the opaque payload is valid; the selected backend validates it and atomically binds the provisioned resource to the trusted `SandboxContext` during `attach`. Any failure rejects the sandbox. +The supervisor validates the descriptor's common fields and produces a `VerifiedTopologyDescriptor`, then resolves its `backend_name` and version without fallback. Verification does not imply that the opaque payload is valid; the selected backend validates it and atomically establishes or locates the resource and binds it to the trusted `SandboxContext` during `attach`. Separated topologies also carry an opaque map of driver-owned immutable resource claims, such as a container ID, pod UID, or VM generation. Control mode presents those claims during authenticated attachment, and boundary mode compares them with its protected configuration before accepting the policy. Any failure rejects the sandbox. ### The lifecycle The contract does not prescribe enforcement mechanisms; it standardizes how the supervisor drives whichever backend a deployment admits. -A backend registers under a `backend_name` and version. The supervisor attaches to the admitted topology and drives the boundary through a fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. +A backend registers under a `backend_name` and version. The supervisor attaches to the admitted topology and drives the boundary through one fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. ```text attach topology + sandbox context -> Bound -> confirm -> Ready -> start_agent -> Running @@ -167,12 +190,23 @@ struct SandboxContext { #[async_trait] trait BoundBoundary: Send { fn network_mediation_source(&self) -> Arc; + fn dns_mediation_source(&self) -> Option>; + fn host_gateway_ip(&self) -> Option; async fn confirm( self: Box, ) -> Result, BackendError>; } +``` +`host_gateway_ip` is the backend's trusted host-side dial target for the +well-known host-gateway aliases. A backend returns it when the mediation +service runs outside the workload boundary and therefore cannot use the +boundary's resolver view; the supervisor preserves the original hostname for +policy, HTTP, and TLS while dialing the backend-provided address. `None` +leaves host-gateway discovery to the supervisor's local environment. + +```rust #[async_trait] trait ReadyBoundary: Send { async fn start_agent( @@ -190,7 +224,7 @@ trait RunningBoundary: Send + Sync { `AgentSpec` carries the complete admitted agent launch specification, including command, arguments, working directory, timeout, and interactive mode. -`SandboxContext` carries the admitted create-time policy. [RFC 0002](../0002-agent-driven-policy-management/README.md) defines how network-policy revisions are proposed and approved. Approved revisions reach the supervisor through the existing [`GetSandboxConfig`](../../proto/sandbox.proto) gateway-supervisor contract, described in the [gateway](../../architecture/gateway.md) and [sandbox](../../architecture/sandbox.md#policy-revision-acknowledgement) architecture. The supervisor makes approved network-policy revisions effective through network mediation. If an approved network-policy revision cannot be loaded, it never becomes effective; the configured rejection posture retains the last valid generation or denies network access until a valid generation is loaded. +`SandboxContext` carries the admitted launch-time policy. [RFC 0002](../0002-agent-driven-policy-management/README.md) defines how network-policy revisions are proposed and approved. Approved revisions reach the supervisor through the existing [`GetSandboxConfig`](../../proto/sandbox.proto) gateway-supervisor contract, described in the [gateway](../../architecture/gateway.md) and [sandbox](../../architecture/sandbox.md#policy-revision-acknowledgement) architecture. The supervisor makes approved network-policy revisions effective through network mediation. If an approved network-policy revision cannot be loaded, it never becomes effective; the configured rejection posture retains the last valid generation or denies network access until a valid generation is loaded. The states have normative meanings: @@ -200,13 +234,15 @@ The states have normative meanings: `confirm` is the pre-launch commit point. The supervisor calls it only after connecting the boundary's network-mediation source to network mediation. The backend confirms standing enforcement for the concrete boundary and may rely on a trusted provisioning-time or out-of-pod signal tied to that boundary's placement, but not on general placement health alone. -`attach` rejects a resource already bound to an active boundary. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `attach` or `confirm`, or the supervisor fails network-mediation initialization. +`attach` rejects a resource already bound to an active boundary or a conflicting launch generation. An idempotent retry resolves the same compatible inactive resource. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `attach` or `confirm`, or the supervisor fails network-mediation initialization. **Standing enforcement** is established independently of a workload process. **Launch-time controls** must be in force before a process executes its first untrusted instruction. Both `start_agent` and `BoundaryExec::exec` enforce this ordering and preserve the provisioned execution environment. `start_agent` is the sole operation that may make the admitted agent runnable. The backend may create or release the process, but workload-controlled code cannot run before `start_agent` applies the required controls. -`RunningBoundary::agent()` returns a handle for the admitted agent process. Processes started through `BoundaryExec` run in the same boundary and have their own process handles. Every workload process remains within the provisioned execution environment. Any exit of the admitted agent ends `Running`; the backend then terminates every remaining workload process within that environment and rejects further runtime operations, except `wait` as defined below. +A separated boundary retains the complete accepted `attach` and `start_agent` inputs for the lifetime of `Running`. If its control process restarts, the replacement replays `attach`, `confirm`, and `start_agent` with the same authenticated boundary identity, resource claims, policy, and launch inputs. The boundary returns the existing process handle without starting a second workload. Any changed input is denied. A main-process stream has one active control owner; transport closure releases that attachment so the replacement control can attach. Compute drivers fence control replacement so old and new control processes do not overlap (for example, a Kubernetes control Deployment uses `Recreate`). + +`RunningBoundary::agent()` returns a handle for the admitted agent process. Processes started through `BoundaryExec` run in the same boundary and have their own process handles. Every workload process remains within the provisioned execution environment. Exit of the admitted agent produces a stable `wait` result and ends that process generation, but it does not implicitly tear down the boundary. The control role may continue serving terminal output, `exec`, and loopback forwarding within the same confirmed boundary until the compute driver or control role explicitly tears the boundary down. Explicit teardown terminates every remaining workload process and rejects new runtime operations. ### Runtime operations @@ -257,11 +293,32 @@ trait NetworkMediationSource: Send + Sync { struct MediatedConnection { stream: BoundaryDuplexStream, binary_identity: Result, + destination: Option, +} + +#[async_trait] +trait DnsMediationSource: Send + Sync { + async fn accept(&self) -> Result; +} + +struct MediatedDnsQuery { + request: Vec, + transport: DnsTransport, + binary_identity: Result, + response: oneshot::Sender, BackendError>>, } ``` `NetworkMediationSource` supplies outbound connections from one boundary to supervisor-owned network mediation. The backend routes all workload egress through that source and authoritatively associates each connection with the boundary without relying solely on workload-provided data. Capture, transport, placement, and coordination are backend-private. +Explicit-proxy transports leave `destination` absent. Transparent transports +capture the original socket destination and supply it before the supervisor +consumes workload bytes. `DnsMediationSource` carries portless DNS exchanges to +the supervisor-owned policy DNS service. It is optional because explicit-proxy +topologies resolve destinations in the supervisor and do not expose workload +DNS. A backend that advertises transparent networking supplies both sources; +DNS or connection-source failure closes that boundary's egress. + Every topology may use the same supervisor-owned mediation libraries or services; the source does not require a backend-specific policy engine. Shared implementations isolate each boundary's state and enforcement. Failure or teardown of one boundary cannot weaken another. Network-mediation unavailability never enables direct egress. @@ -289,13 +346,13 @@ Binary identity is mandatory conformance: RFC 0002 makes it part of the outbound ### The supervisor sequence -The logical supervisor resolves `backend_name` and version through a trusted implementation registry. Adding a backend adds an implementation and registration, not branches in lifecycle, proxy, SSH, or session code. Delegated transport and coordination remain backend-private. +The logical supervisor resolves `backend_name` and version through a trusted implementation registry. A separated topology selects the reusable remote backend and supplies its standardized endpoint in the opaque descriptor. Adding a driver adds provisioning and endpoint construction in that driver's crate, not branches in lifecycle, proxy, SSH, session, or control-mode code. The supervisor runs the same sequence for every backend: -1. Obtain the `TopologyDescriptor` and trusted `SandboxContext`. +1. Obtain the trusted `TopologyDescriptor` and `SandboxContext` selected by admission. 2. Verify the descriptor and resolve its `backend_name` and version without fallback. -3. Call `attach` to obtain `Bound`. +3. Call `attach` to establish or locate the resource, bind it, and obtain `Bound`. 4. Connect the boundary's `NetworkMediationSource` to network mediation. 5. Call `confirm` to obtain `Ready`, then `start_agent` to obtain `Running`. 6. Use the returned runtime handles for agent wait, `exec`, and port forwarding while network mediation consumes outbound connections. @@ -312,18 +369,20 @@ enum BackendErrorKind { Invalid, Denied, Unavailable, Unsupported, Failed, Termi `Invalid` covers descriptor, version, and backend mismatches; `Denied` covers authenticated attachment rejection; `Unavailable` covers transient inability to serve an operation; `Unsupported` identifies an optional operation the selected backend does not implement; `Failed` covers other backend faults; and `Terminated` reports boundary or workload termination, or an operation against an inactive boundary. An error never advances the lifecycle or authorizes an operation, and backend selection never falls back. -A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per provisioned topology. If it does not return `Bound`, the topology is reclaimed rather than reused. +An `Unsupported` error variant reports that the selected backend does not implement an optional contract operation; it maps to the `Unavailable` kind for status purposes and never weakens a mandatory conformance requirement. + +A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per orchestration attempt. Attach-time establishment uses sandbox identity and launch generation as an idempotency key. If the operation does not return `Bound`, the compute driver reclaims the topology rather than reusing an ambiguous resource. Failures resolve as follows: -- an `attach` or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the driver to reclaim the topology; -- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the driver reclaims the topology; +- an `attach` or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the compute driver to reclaim the topology; +- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the compute driver reclaims the topology; - if `exec` or port-forward `connect` fails, the backend terminates any process or closes any connection created by that attempt while the boundary otherwise remains active; - after `Running`, supervisor or enforcement loss follows invariant 6; when enforcement loss ends the agent, `BoundaryProcess::wait` fails with `BackendErrorKind::Terminated` where process-exit observation survives; - network-mediation errors yield no authorized connection and do not by themselves end `Running`; and - retained runtime handles and the network-mediation source reject new operations whenever the boundary ends, except `BoundaryProcess::wait` where the backend can still return its stable result. -Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the compute driver reclaims or deprovisions the topology. If normal backend cleanup is unavailable, the compute driver uses the topology's trusted cleanup path to terminate the execution environment and invalidate the binding before reclaim or reuse. On normal agent exit, `BoundaryProcess::wait` returns the stable exit status. A retained `wait` result may outlive teardown. +Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the compute driver reclaims or deprovisions the topology. If normal backend cleanup is unavailable, the compute driver uses the topology's trusted cleanup path to terminate the execution environment and invalidate the binding before reclaim or reuse. Normal agent exit alone does not end the boundary: `BoundaryProcess::wait` returns the stable exit status while the confirmed access plane remains available until explicit teardown. A retained `wait` result may outlive teardown. ### Topologies @@ -331,11 +390,12 @@ The contract fixes the roles; a topology fixes their placement. Components may b ## Implementation plan -This RFC defines the contract; implementation lands in three phases: +This RFC defines the contract; implementation lands in four phases: 1. **Contract.** Add the common types, descriptor handling, registry, and explicit backend selection from deployment configuration. -2. **Co-located backend.** Implement the co-located backend behind a deployment flag and route agent launch, egress interception, the network-mediation source, SSH, `exec`, and forwarding through it without changing behavior. -3. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus descriptor verification, lifecycle ordering, runtime operations, and failure semantics. Make the co-located backend the default after parity validation. Parity covers the agent, binary identity, SSH, `exec`, and forwarding paths; enablement also closes the in-pod egress gaps pinned in [codebase-grounding.md](./codebase-grounding.md), which parity alone would preserve. +2. **Shared supervisor modes.** Add the versioned boundary protocol, reusable remote backend, and the `control` and `boundary` entrypoints to `openshell-sandbox`. +3. **Driver adoption.** Have VM, Docker, and Kubernetes provision protected boundary configs and topology descriptors in their existing driver crates. No adoption may require a shared-supervisor change; that constraint is the abstraction test. +4. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus descriptor verification, lifecycle ordering, runtime operations, compute-driver-owned cleanup, and failure semantics. Existing placements remain outside this contract until their backend is implemented and admitted; they do not claim conformance. Delegated backends remain separate design and implementation work. @@ -343,7 +403,8 @@ Existing placements remain outside this contract until their backend is implemen | Risk | Mitigation | |---|---| -| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep the responsibility boundary explicit: the compute driver owns, provisions, and deprovisions the topology; the backend binds and operates the active boundary. The same component may implement both roles. | +| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep placement, preparation, durable deletion, reconciliation, and endpoint selection in the compute driver; keep enforcement sequencing in control mode and boundary-local observation in boundary mode. Generic supervisor code never imports a concrete driver. | +| Attach-time establishment could leave a partial resource after supervisor failure. | Make establishment idempotent by sandbox launch generation, label resources for compute-driver reconciliation, and do not start untrusted code before `attach` and `confirm` complete. | | Contract conformance could be mistaken for equivalent isolation across topologies. | Treat conformance as behavioral, not as a security-strength rating. Document and validate each topology's actual containment and reject policy it cannot enforce. | | Shared backend or network-mediation components concentrate privilege and failure impact. | Isolate state, connection attribution, enforcement, and control authority per boundary. Failure of one boundary must not weaken another or enable direct egress. | | The mandatory contract may exclude otherwise useful but incomplete backends. | Keep the network-mediation source, binary identity, process control, `exec`, and port forwarding mandatory. An incomplete backend does not claim conformance or silently degrade. | @@ -358,17 +419,17 @@ OpenShell could keep the current in-pod design and add topology-specific supervi Doing nothing avoids a new interface, but retains privileged boundary construction beside the workload. Implementing each delegated topology as a one-off supervisor change moves that privilege for one placement but accretes topology-specific supervisor behavior. The proposed contract instead keeps one supervisor lifecycle while allowing the topology to change. -### Extend the compute-driver contract +### Require every resource to exist before attach -The compute driver could own both provisioning and active-boundary operation. +The compute driver could always create the concrete resource before the supervisor calls `attach`. -This is natural for topologies such as MXC, and the same component may implement both responsibilities. The interfaces remain distinct because they serve different callers and lifecycles: the gateway uses the compute driver to provision and deprovision resources, while the supervisor uses the Isolation Backend to operate an active boundary. Combining them would couple runtime policy, identity, network mediation, and process operations to the gateway-facing driver API. +This remains natural for controller-driven systems such as Kubernetes. Requiring it everywhere prevents a local backend from establishing host listeners and enforcement state before a runtime creates the workload. Allowing a trusted topology descriptor to carry prepared inputs preserves the single `attach` contract while allowing security-sensitive establishment to remain atomic with binding. The compute driver still owns deletion and reconciliation. -### Start with a remote backend service +### Give each driver its own remote control service -The contract could be expressed as a gRPC service or plugin ABI rather than an in-process Rust contract. [RFC 0001](../0001-core-architecture/README.md) chose gRPC for its gateway-facing drivers, so the question applies here. +Each separated driver could define a private gRPC service, guest agent, or runtime-specific plugin ABI behind its `IsolationBackend` implementation. -The callers differ. A gateway driver is a control-plane peer with its own release cycle, while the Isolation Backend is driven by the supervisor that operates the boundary, and the co-located topology needs no transport at all. Starting in-process serves that case directly and lets delegated implementations carry their own transport behind the same interface. A transport-bearing surface is not precluded: it is versioned contract surface, added when a concrete delegated backend requires it. +That would hide transport differences from the Rust trait, but it would duplicate lifecycle, authentication, process streaming, signaling, forwarding, and binary-identity semantics across Docker, Kubernetes, and VM implementations. The proposed versioned boundary protocol standardizes those semantics once. Drivers still choose and provision Unix socket, authenticated TCP, vsock, or adapter transport and bind their own immutable resource claims. ### Standardize topology and capabilities @@ -381,10 +442,11 @@ That would make known deployments explicit, but it would also encode current top - **Driver-backed subsystems (CRI/CNI/CSI).** Kubernetes factors runtime, networking, and storage into pluggable driver contracts so the orchestrator drives one interface while implementations vary. RFC 0001 describes OpenShell's other subsystems the same way; this RFC specifies the one it left open: isolation. - **Istio privilege placement.** Init-sidecar and node-agent modes demonstrate that network setup can move without changing the policy data path. OpenShell keeps its identity-aware proxy. - **CRI exec/attach/port-forward.** `exec` and `connect` follow CRI's `Exec` and `PortForward` shape; lifecycle and network mediation remain OpenShell-specific. +- **[OCI seccomp listener handoff](https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#seccomp).** The runtime specification lets a runtime send a seccomp notification FD and process state to a host Unix listener. It demonstrates why some local enforcement must exist before workload creation and informs attach-time boundary establishment. ## Open questions -None. +None for this revision. New common fields require evidence from a concrete backend and a protocol-version change when compatibility cannot be preserved. ## Appendix: codebase grounding diff --git a/rfc/0012-isolation-backend/topology-matrix.md b/rfc/0012-isolation-backend/topology-matrix.md index 8dc14d84ba..85335be545 100644 --- a/rfc/0012-isolation-backend/topology-matrix.md +++ b/rfc/0012-isolation-backend/topology-matrix.md @@ -6,18 +6,29 @@ kernel; it does not select a deployment or establish conformance. ## Representative placements -| Pattern | Logical supervisor and network-mediation placement | Backend placement | Workload-kernel relationship | Topology status | +| Pattern | Control and network-mediation placement | Boundary-mode placement | Workload-kernel relationship | Topology status | |---|---|---|---|---| -| **Co-located/in-pod** | With the workload | In the supervisor process | Trusted components share the workload's host, guest, or application kernel, depending on the runtime | Placement implemented (original topology) | -| **Same-pod composite** | Spans the workload-local supervisor process and, when used, a network-mediation sidecar | In the workload-local supervisor process | Components share the workload's kernel | Placement implemented (#2076) | -| **Delegated backend components** | With the workload and any delegated mediation component | A node or remote helper establishes some controls behind a workload-local backend | Depends on which trusted components remain with the workload | Placement proposed (#2606) | -| **Driver-hosted/shared service** | With the compute driver or another trusted service; no in-sandbox supervisor process is required | May be co-located with the logical supervisor; one host may operate many isolated boundaries | Depends on the workload runtime | Placement proposed | +| **Co-located/in-pod** | With the workload; the legacy in-process backend may omit boundary mode | Same process when used | Trusted components share the workload's host kernel | Placement implemented (original topology) | +| **Kubernetes proxy pod** | Trusted control pod | Boundary-mode workload entrypoint owning the workload PID and network namespaces | Shared cluster-node kernel; pod security boundaries separate control from workload | Implemented in #3144; requires a conforming NetworkPolicy CNI and trusted namespace | +| **Docker** | Gateway host | Trusted container entrypoint sharing the workload container's PID and network namespaces | Shared host kernel | Implemented in #2965 | +| **MicroVM** | Gateway host | Guest PID 1 | Boundary mode shares the guest kernel; control is kernel-separated | Implemented in #2945 | + +The Kubernetes proxy-pod topology uses the same boundary protocol as Docker and +VM, with per-boundary TLS because the connection traverses the pod network. +Kubernetes-specific code provisions the workload fence, pair labels, boundary +Service, control Deployment, immutable bootstrap Secret, and stable +namespace/Sandbox/Deployment/NetworkPolicy claims. The workload pod has no +direct egress; attributed proxy streams cross the TLS channel and hostname +resolution occurs on the control side. Admission requires an explicitly acknowledged conforming CNI and a +namespace in which untrusted principals cannot create pods, mutate pair labels, +or read bootstrap Secrets. Pod readiness or the existence of a `NetworkPolicy` +object alone does not prove enforcement. ## Durable rules - Every active boundary has one verified descriptor, one trusted - `SandboxContext`, and at most one logical supervisor, which may span multiple - coupled processes. + `SandboxContext`, one control role, and at most one boundary role. Those + processes form one logical supervisor. - Physical processes and listeners may be shared, but lifecycle state, policy, binary identity, enforcement, and cleanup remain isolated per boundary. - Moving a privileged component does not itself provide kernel separation.