From 3a4f61b676db6f977898c3adc49b206ccc0535c9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 2 Sep 2026 17:10:17 -0700 Subject: [PATCH 01/37] feat(sandbox): add control and boundary supervisor modes Signed-off-by: Drew Newberry --- Cargo.lock | 7 + architecture/compute-runtimes.md | 20 +- architecture/sandbox.md | 39 +- crates/openshell-core/src/sandbox_env.rs | 7 + .../openshell-isolation-interface/Cargo.toml | 6 + .../src/boundary_protocol.rs | 849 ++++++++++ .../openshell-isolation-interface/src/lib.rs | 4 + .../src/remote.rs | 944 +++++++++++ crates/openshell-sandbox/Cargo.toml | 5 + .../openshell-sandbox/src/boundary_server.rs | 1338 ++++++++++++++++ crates/openshell-sandbox/src/lib.rs | 189 ++- crates/openshell-sandbox/src/main.rs | 156 +- .../openshell-supervisor-network/src/lib.rs | 1 + .../openshell-supervisor-network/src/opa.rs | 1 + .../src/policy_dns/runtime.rs | 70 + .../src/policy_dns/store.rs | 11 + .../openshell-supervisor-network/src/proxy.rs | 548 ++++++- .../src/proxy/egress.rs | 2 + .../openshell-supervisor-network/src/run.rs | 24 + .../src/spiffe_endpoint.rs | 18 + .../src/boundary_exec.rs | 6 +- .../src/boundary_io.rs | 12 +- .../src/delegated.rs | 501 ++++++ .../openshell-supervisor-process/src/lib.rs | 1 + .../src/netns/mod.rs | 1312 ++++++++++++--- .../src/process.rs | 35 +- .../openshell-supervisor-process/src/run.rs | 69 +- .../openshell-supervisor-process/src/ssh.rs | 1400 ++++++----------- .../src/supervisor_session.rs | 112 +- deploy/docker/Dockerfile.supervisor | 14 +- rfc/0012-isolation-backend/README.md | 147 +- rfc/0012-isolation-backend/topology-matrix.md | 20 +- 32 files changed, 6426 insertions(+), 1442 deletions(-) create mode 100644 crates/openshell-isolation-interface/src/boundary_protocol.rs create mode 100644 crates/openshell-isolation-interface/src/remote.rs create mode 100644 crates/openshell-sandbox/src/boundary_server.rs create mode 100644 crates/openshell-supervisor-network/src/spiffe_endpoint.rs create mode 100644 crates/openshell-supervisor-process/src/delegated.rs diff --git a/Cargo.lock b/Cargo.lock index ac8d20e2d4..bd89c34f9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4176,7 +4176,11 @@ name = "openshell-isolation-interface" version = "0.0.0" dependencies = [ "async-trait", + "libc", "openshell-core", + "serde", + "serde_json", + "thiserror 2.0.18", "tokio", ] @@ -4281,12 +4285,15 @@ dependencies = [ name = "openshell-sandbox" version = "0.0.0" dependencies = [ + "base64 0.22.1", "clap", "futures", + "libc", "miette", "nix 0.29.0", "openshell-core", "openshell-extension-core", + "openshell-isolation-interface", "openshell-ocsf", "openshell-policy", "openshell-supervisor-middleware", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e1e731a0ce..a69406a39f 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -1,10 +1,11 @@ # Compute Runtimes Compute runtimes create, stop, start, delete, and watch sandbox workloads for the -gateway. Supervisor-controlled runtimes start a workload that runs the -`openshell-sandbox` supervisor, which enforces the sandbox contract locally. -Driver-controlled runtimes apply the canonical sandbox policy while -provisioning and report workload readiness directly. +gateway. They do not replace sandbox policy enforcement. A runtime either starts +the co-located `openshell-sandbox` supervisor or provisions its RFC 0012 +`--mode=control` and `--mode=boundary` placements. Runtimes without the standard +supervisor apply the canonical sandbox policy while provisioning and report +workload readiness directly. ## Driver Contract @@ -12,9 +13,14 @@ Each runtime receives a sandbox spec and canonical policy from the gateway and is responsible for: - Selecting the sandbox image. -- For supervisor-controlled runtimes, injecting sandbox identity and gateway - callback configuration, supplying callback credentials, and providing the - supervisor binary or image. +- Injecting sandbox identity and gateway callback configuration. +- Supplying TLS or secret material for supervisor callbacks. +- Providing the supervisor binary or image in the workload. +- Provisioning protected control and boundary configs plus a private Unix socket, + authenticated TCP, or vsock transport when the supervisor is separated. + Runtime-specific code supplies immutable resource claims and transport + coordinates; the shared boundary protocol supplies lifecycle, exec, signaling, + forwarding, and binary identity semantics. - For runtimes without the standard supervisor, validating and applying the canonical policy before launching the workload. - Forwarding the exact canonical main-process argv and TTY mode without shell diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 055ef7e4a3..fc5957deb3 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -1,8 +1,7 @@ # Sandbox -A sandbox is the runtime boundary where agent code executes. It is created by a -compute runtime and managed inside the workload by `openshell-sandbox`, the -sandbox supervisor. +A sandbox is the runtime boundary where agent code executes. A compute runtime +creates it, and `openshell-sandbox` operates it as one logical supervisor. ## Runtime Model @@ -21,18 +20,38 @@ container-granted capabilities. This is fail-closed: the supervisor retains aborts unless the bounding set ends up empty. A `setpcap` `EPERM` is tolerated only when the set is already empty; any other outcome fails the spawn. +Separated runtimes split the same logical supervisor into two placements: + +| Mode | Placement and authority | +|---|---| +| `openshell-sandbox --mode=control` | Runs outside the workload boundary. Owns gateway credentials, the admitted policy, RFC 0012 lifecycle, network-policy decisions, SSH, and gateway relays. | +| `openshell-sandbox --mode=boundary --boundary-config ` | Runs inside or adjacent to the workload. Owns process launch and observation, exec signaling, PTY control, loopback forwarding, egress capture, and binary identity. It has no gateway credentials or independent policy authority. | + +The compute driver provisions the two protected configurations and their private +Unix-socket, authenticated TCP, or vsock transport. All configurations carry the +same boundary ID, bootstrap credential, protocol version, and immutable driver resource claims. +The shared remote backend authenticates each request and rejects a resource-claim +mismatch. Driver crates do not appear in generic supervisor lifecycle, network, +SSH, or session code. + +For cross-UID Unix-socket placements, the driver-owned parent directory limits +socket reachability and the bootstrap credential authenticates every request. +Boundary mode can re-own a bind-mounted config as root-only before starting the +workload when its host ownership could otherwise coincide with the workload UID. + ## Startup Flow 1. The compute runtime starts the workload with sandbox identity, callback endpoint, TLS or secret material, image metadata, and initial command. -2. The supervisor loads policy and runtime settings from local files or the +2. Control mode loads policy and runtime settings from local files or the gateway, depending on mode. -3. It prepares filesystem access, process restrictions, network namespace - routing, trust stores, provider credential resolution, and inference routes. -4. It launches the persisted canonical main-process argv and retains its PTY - or pipes in the main-session multiplexer. -5. It starts the policy proxy and local SSH server. -6. It opens a supervisor session back to the gateway for connect, exec, file +3. The isolation backend attaches and confirms standing enforcement. Boundary + mode prepares boundary-local filesystem, process, and egress controls. +4. Control mode starts the admitted main process through the backend. Boundary + mode applies launch-time controls before its first untrusted instruction. +5. Control mode starts the policy proxy and local SSH server. Exec and loopback + streams cross the shared boundary protocol when the modes are separated. +6. Control mode opens a supervisor session back to the gateway for connect, exec, file sync, config polling, and log push. ## Isolation Layers diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 99ac55fe6e..1feddc743d 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -122,6 +122,13 @@ pub const TELEMETRY_ENABLED: &str = "OPENSHELL_TELEMETRY_ENABLED"; /// `"sidecar"`; the default combined supervisor path omits it. pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY"; +/// The isolation backend admitted by the deployment configuration (RFC 0012). +/// +/// Delivered on a channel separate from the topology descriptor so descriptor +/// verification against the admitted backend is not self-referential. Required +/// whenever a topology descriptor is supplied. +pub const ADMITTED_ISOLATION_BACKEND: &str = "OPENSHELL_ADMITTED_ISOLATION_BACKEND"; + /// Network enforcement backend selected by the compute driver. pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE"; diff --git a/crates/openshell-isolation-interface/Cargo.toml b/crates/openshell-isolation-interface/Cargo.toml index 647f19dad4..e86ba7e5e8 100644 --- a/crates/openshell-isolation-interface/Cargo.toml +++ b/crates/openshell-isolation-interface/Cargo.toml @@ -13,8 +13,14 @@ repository.workspace = true [dependencies] openshell-core = { path = "../openshell-core", default-features = false } async-trait = "0.1" +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true } +[target.'cfg(unix)'.dependencies] +libc = "0.2" + [dev-dependencies] tokio = { workspace = true } diff --git a/crates/openshell-isolation-interface/src/boundary_protocol.rs b/crates/openshell-isolation-interface/src/boundary_protocol.rs new file mode 100644 index 0000000000..efd233a5ab --- /dev/null +++ b/crates/openshell-isolation-interface/src/boundary_protocol.rs @@ -0,0 +1,849 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Versioned control protocol shared by every remote isolation boundary. +//! +//! Drivers choose and provision the transport, but they do not redefine the +//! process lifecycle, streaming, identity, or authentication messages. The +//! control and boundary roles exchange these length-delimited JSON frames over +//! a private Unix socket, authenticated TCP connection, or virtio-vsock stream. + +use std::fmt; +use std::io; +use std::io::{Read, Write}; +use std::path::PathBuf; + +use crate::AgentSpec; +use crate::contract::Sha256Digest; +use crate::contract::{ + BackendError, BinaryIdentity, BoundaryExitStatus, BoundarySignal, ExecSpec, INTERFACE_VERSION, + ResolveError, TopologyDescriptor, +}; +use openshell_core::policy::{ + FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkMode, NetworkPolicy, + ProcessPolicy, ProxyPolicy, SandboxPolicy, +}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +pub const MAX_CONTROL_FRAME_BYTES: usize = 1024 * 1024; +pub const STREAM_STDIN: u8 = 0; +pub const STREAM_STDOUT: u8 = 1; +pub const STREAM_STDERR: u8 = 2; +pub const STREAM_EXIT: u8 = 3; +pub const STREAM_STDIN_CLOSED: u8 = 4; +pub const MAX_STREAM_FRAME_BYTES: usize = 64 * 1024; +/// Version of the driver-to-supervisor boundary descriptor and config shape. +pub const BOUNDARY_PROTOCOL_VERSION: u32 = 1; + +/// Control-side endpoint for a driver-provisioned boundary. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoundaryTransport { + /// Private Unix socket, including a host endpoint mapped to guest vsock. + Unix { socket_path: PathBuf }, + /// Authenticated TCP endpoint, normally scoped by the runtime network. + Tcp { address: std::net::SocketAddr }, + /// Linux host `AF_VSOCK` connection to a guest boundary. + Vsock { guest_cid: u32, control_port: u32 }, +} + +/// Boundary-side listener provisioned by a compute driver. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoundaryListener { + /// Private Unix socket shared with a host or sidecar transport adapter. + Unix { socket_path: PathBuf }, + /// TCP listener address. An unspecified IP is valid for the boundary-side + /// bind; the control-side transport must contain a concrete dial target. + Tcp { address: std::net::SocketAddr }, + /// Guest `AF_VSOCK` listener. + Vsock { control_port: u32 }, +} + +/// Workload identity selected by the compute driver and resolved at the +/// boundary before the first workload instruction executes. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoundaryAgentIdentity { + /// A platform-selected numeric identity, as used by VM and Kubernetes. + Resolved { uid: u32, gid: u32 }, + /// The image's raw OCI `Config.User` declaration, as used by Docker. + OciUser { declaration: String }, + /// No driver-authoritative identity metadata is available. + None, +} + +/// Protected control descriptor delivered to `--mode=control`. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryTopology { + /// Wire-format version. Must match [`BOUNDARY_PROTOCOL_VERSION`]. + pub protocol_version: u32, + /// Stable identity of the boundary, normally the sandbox ID. + pub boundary_id: String, + /// Driver-provisioned control endpoint. + pub transport: BoundaryTransport, + /// Trusted dial target for well-known host-gateway aliases, when the + /// network supervisor cannot use the boundary's resolver view. + #[serde(default)] + pub host_gateway_ip: Option, + /// Driver-specific immutable resource coordinates bound at attach (for + /// example pod UID, VM generation, or container ID). + #[serde(default)] + pub resource_claims: std::collections::BTreeMap, + /// Per-boundary authentication secret; never exposed to workload code. + pub bootstrap_token: String, +} + +impl fmt::Debug for BoundaryTopology { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BoundaryTopology") + .field("protocol_version", &self.protocol_version) + .field("boundary_id", &self.boundary_id) + .field("transport", &self.transport) + .field("host_gateway_ip", &self.host_gateway_ip) + .field("resource_claims", &self.resource_claims) + .field("bootstrap_token", &"") + .finish() + } +} + +impl BoundaryTopology { + /// Encode this topology as the shared RFC 0012 descriptor admitted for + /// `backend_name`. + pub fn descriptor( + &self, + backend_name: impl Into, + ) -> Result { + let payload = serde_json::to_vec(self) + .map_err(|error| BackendError::Descriptor(format!("encode topology: {error}")))?; + Ok(TopologyDescriptor { + version: INTERFACE_VERSION, + backend_name: backend_name.into(), + payload, + }) + } +} + +/// Protected configuration delivered to `--mode=boundary`. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryConfig { + /// Wire-format version. Must match [`BOUNDARY_PROTOCOL_VERSION`]. + pub protocol_version: u32, + /// Stable identity expected in every authenticated request. + pub boundary_id: String, + /// Per-boundary authentication secret. + pub bootstrap_token: String, + /// Re-own this config as boundary-root-only after it has been decoded and + /// before any workload process can start. Drivers use this when the + /// protected delivery file's host owner may match the workload UID. + #[serde(default)] + pub protect_config_file: bool, + /// Driver-provisioned listener. + pub listener: BoundaryListener, + /// Immutable coordinates the boundary requires from the control-side + /// topology descriptor before accepting attachment. + #[serde(default)] + pub resource_claims: std::collections::BTreeMap, + /// Driver-selected identity for the admitted workload. + pub agent_identity: BoundaryAgentIdentity, + /// Absolute, driver-owned helper runtime used for namespace setup. + pub trusted_runtime_root: PathBuf, + /// Driver-resolved environment exposed only to workload processes. + #[serde(default)] + pub child_env: std::collections::HashMap, +} + +impl fmt::Debug for BoundaryConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BoundaryConfig") + .field("protocol_version", &self.protocol_version) + .field("boundary_id", &self.boundary_id) + .field("bootstrap_token", &"") + .field("protect_config_file", &self.protect_config_file) + .field("listener", &self.listener) + .field("resource_claims", &self.resource_claims) + .field("agent_identity", &self.agent_identity) + .field("trusted_runtime_root", &self.trusted_runtime_root) + .field("child_env_keys", &self.child_env.keys().collect::>()) + .finish() + } +} + +impl BoundaryConfig { + /// Serialize the protected driver-owned boundary configuration. + pub fn encode(&self) -> Result, BackendError> { + serde_json::to_vec(self) + .map_err(|error| BackendError::Descriptor(format!("encode boundary config: {error}"))) + } +} + +/// Validate driver-specific immutable coordinates before a boundary binds them. +/// +/// Claim values are opaque to the common protocol, but empty or +/// whitespace-bearing identifiers cannot safely distinguish runtime objects. +pub fn validate_resource_claims( + claims: &std::collections::BTreeMap, +) -> Result<(), BackendError> { + for (key, value) in claims { + if key.is_empty() || key.chars().any(char::is_whitespace) { + return Err(BackendError::Descriptor( + "boundary resource-claim keys must be non-empty and contain no whitespace" + .to_string(), + )); + } + if value.is_empty() || value.chars().any(char::is_whitespace) { + return Err(BackendError::Descriptor(format!( + "boundary resource claim {key:?} must be non-empty and contain no whitespace" + ))); + } + } + Ok(()) +} + +pub async fn write_stream_frame( + writer: &mut (impl AsyncWrite + Unpin), + channel: u8, + payload: &[u8], +) -> io::Result<()> { + if payload.len() > MAX_STREAM_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "boundary stream frame exceeds limit", + )); + } + writer.write_u8(channel).await?; + writer + .write_u32(payload.len().try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "boundary stream frame length overflow", + ) + })?) + .await?; + writer.write_all(payload).await?; + writer.flush().await +} + +pub async fn read_stream_frame( + reader: &mut (impl AsyncRead + Unpin), +) -> io::Result)>> { + let channel = match reader.read_u8().await { + Ok(channel) => channel, + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + Err(error) => return Err(error), + }; + let declared = reader.read_u32().await? as usize; + if declared > MAX_STREAM_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("boundary stream frame is too large: {declared} bytes"), + )); + } + let mut payload = vec![0; declared]; + reader.read_exact(&mut payload).await?; + Ok(Some((channel, payload))) +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RequestEnvelope { + pub request_id: u64, + pub boundary_id: String, + pub bootstrap_token: String, + pub request: Request, +} + +impl fmt::Debug for RequestEnvelope { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RequestEnvelope") + .field("request_id", &self.request_id) + .field("boundary_id", &self.boundary_id) + .field("bootstrap_token", &"") + .field("request", &self.request) + .finish() + } +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "operation", rename_all = "snake_case")] +pub enum Request { + Attach { + policy: Box, + resource_claims: std::collections::BTreeMap, + }, + Confirm, + StartAgent { + sandbox_id: String, + spec: AgentSpecWire, + policy: Box, + ca_cert: Option>, + ca_bundle: Option>, + provider_env: std::collections::HashMap, + }, + Wait { + process_id: String, + }, + Signal { + process_id: String, + signal: SignalWire, + }, + Terminate { + process_id: String, + }, + Exec { + spec: ExecSpecWire, + }, + ExecSignal { + process_id: String, + signal: SignalWire, + }, + Resize { + process_id: String, + cols: u16, + rows: u16, + }, + PortForward { + host: std::net::IpAddr, + port: u16, + }, + AcceptNetwork, +} + +impl fmt::Debug for Request { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Attach { + policy: _, + resource_claims, + } => formatter + .debug_struct("Attach") + .field("policy", &"") + .field("resource_claims", resource_claims) + .finish(), + Self::Confirm => formatter.write_str("Confirm"), + Self::StartAgent { + sandbox_id, + spec, + policy: _, + ca_cert, + ca_bundle, + provider_env, + } => formatter + .debug_struct("StartAgent") + .field("sandbox_id", sandbox_id) + .field("spec", spec) + .field("policy", &"") + .field("ca_cert_present", &ca_cert.is_some()) + .field("ca_bundle_present", &ca_bundle.is_some()) + .field( + "provider_env_keys", + &provider_env.keys().collect::>(), + ) + .finish(), + Self::Wait { process_id } => formatter + .debug_struct("Wait") + .field("process_id", process_id) + .finish(), + Self::Signal { process_id, signal } => formatter + .debug_struct("Signal") + .field("process_id", process_id) + .field("signal", signal) + .finish(), + Self::Terminate { process_id } => formatter + .debug_struct("Terminate") + .field("process_id", process_id) + .finish(), + Self::Exec { spec } => formatter.debug_tuple("Exec").field(spec).finish(), + Self::ExecSignal { process_id, signal } => formatter + .debug_struct("ExecSignal") + .field("process_id", process_id) + .field("signal", signal) + .finish(), + Self::Resize { + process_id, + cols, + rows, + } => formatter + .debug_struct("Resize") + .field("process_id", process_id) + .field("cols", cols) + .field("rows", rows) + .finish(), + Self::PortForward { host, port } => formatter + .debug_struct("PortForward") + .field("host", host) + .field("port", port) + .finish(), + Self::AcceptNetwork => formatter.write_str("AcceptNetwork"), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResponseEnvelope { + pub request_id: u64, + pub response: Response, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "snake_case")] +pub enum Response { + Attached, + Confirmed, + Started { process_id: String }, + Exited { status: ExitStatusWire }, + Signaled, + Terminated, + ExecStarted { process_id: String, pty: bool }, + Resized, + PortConnected, + NetworkConnected { identity: BinaryIdentityWire }, + Error { kind: String, message: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BinaryIdentityWire { + pub binary_path: Option, + pub binary_digest: Option, + pub ancestors: Vec, + pub cmdline_paths: Vec, + pub resolve_error: Option, +} + +impl From> for BinaryIdentityWire { + fn from(identity: Result) -> Self { + match identity { + Ok(identity) => Self { + binary_path: Some(identity.binary_path), + binary_digest: identity.binary_digest.map(|digest| digest.to_string()), + ancestors: identity.ancestors, + cmdline_paths: identity.cmdline_paths, + resolve_error: None, + }, + Err(error) => Self { + binary_path: None, + binary_digest: None, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + resolve_error: Some(error.to_string()), + }, + } + } +} + +impl BinaryIdentityWire { + pub fn into_result(self) -> Result { + if let Some(error) = self.resolve_error { + return Err(ResolveError::Failed(error)); + } + let binary_path = self.binary_path.ok_or_else(|| { + ResolveError::Failed("boundary identity omitted binary path".to_string()) + })?; + let binary_digest = self + .binary_digest + .map(|digest| digest.parse::()) + .transpose()?; + Ok(BinaryIdentity { + binary_path, + binary_digest, + ancestors: self.ancestors, + cmdline_paths: self.cmdline_paths, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExecSpecWire { + pub program: String, + pub args: Vec, + pub env: Vec<(String, String)>, + pub workdir: Option, + pub pty: bool, +} + +impl From for ExecSpecWire { + fn from(spec: ExecSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + env: spec.env, + workdir: spec.workdir, + pty: spec.pty, + } + } +} + +impl From for ExecSpec { + fn from(spec: ExecSpecWire) -> Self { + Self { + program: spec.program, + args: spec.args, + env: spec.env, + workdir: spec.workdir, + pty: spec.pty, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentSpecWire { + pub program: String, + pub args: Vec, + pub workdir: Option, + pub timeout_secs: u64, + pub interactive: bool, +} + +impl From for AgentSpecWire { + fn from(spec: AgentSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + workdir: spec.workdir, + timeout_secs: spec.timeout_secs, + interactive: spec.interactive, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxPolicyWire { + pub version: u32, + pub read_only: Vec, + pub read_write: Vec, + pub include_workdir: bool, + pub network: NetworkModeWire, + pub proxy_addr: Option, + pub landlock: LandlockCompatibilityWire, + pub run_as_user: Option, + pub run_as_group: Option, +} + +impl From for SandboxPolicyWire { + fn from(policy: SandboxPolicy) -> Self { + // Exhaustively destructure the policy so adding a `SandboxPolicy` + // field is a compile error here instead of a silently dropped field + // across the host-to-guest trust boundary. + let SandboxPolicy { + version, + filesystem, + network, + landlock, + process, + } = policy; + let FilesystemPolicy { + read_only, + read_write, + include_workdir, + } = filesystem; + let NetworkPolicy { mode, proxy } = network; + let LandlockPolicy { compatibility } = landlock; + let ProcessPolicy { + run_as_user, + run_as_group, + } = process; + Self { + version, + read_only, + read_write, + include_workdir, + network: NetworkModeWire::from(mode), + proxy_addr: proxy.and_then(|proxy| proxy.http_addr), + landlock: LandlockCompatibilityWire::from(compatibility), + run_as_user, + run_as_group, + } + } +} + +impl From for SandboxPolicy { + fn from(policy: SandboxPolicyWire) -> Self { + let proxy = matches!(policy.network, NetworkModeWire::Proxy).then_some(ProxyPolicy { + http_addr: policy.proxy_addr, + }); + Self { + version: policy.version, + filesystem: FilesystemPolicy { + read_only: policy.read_only, + read_write: policy.read_write, + include_workdir: policy.include_workdir, + }, + network: NetworkPolicy { + mode: policy.network.into(), + proxy, + }, + landlock: LandlockPolicy { + compatibility: policy.landlock.into(), + }, + process: ProcessPolicy { + run_as_user: policy.run_as_user, + run_as_group: policy.run_as_group, + }, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NetworkModeWire { + Block, + Proxy, + Allow, +} + +impl From for NetworkModeWire { + fn from(mode: NetworkMode) -> Self { + match mode { + NetworkMode::Block => Self::Block, + NetworkMode::Proxy => Self::Proxy, + NetworkMode::Allow => Self::Allow, + } + } +} + +impl From for NetworkMode { + fn from(mode: NetworkModeWire) -> Self { + match mode { + NetworkModeWire::Block => Self::Block, + NetworkModeWire::Proxy => Self::Proxy, + NetworkModeWire::Allow => Self::Allow, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LandlockCompatibilityWire { + BestEffort, + HardRequirement, +} + +impl From for LandlockCompatibilityWire { + fn from(compatibility: LandlockCompatibility) -> Self { + match compatibility { + LandlockCompatibility::BestEffort => Self::BestEffort, + LandlockCompatibility::HardRequirement => Self::HardRequirement, + } + } +} + +impl From for LandlockCompatibility { + fn from(compatibility: LandlockCompatibilityWire) -> Self { + match compatibility { + LandlockCompatibilityWire::BestEffort => Self::BestEffort, + LandlockCompatibilityWire::HardRequirement => Self::HardRequirement, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignalWire { + Term, + Kill, + Int, + Hup, +} + +impl From for SignalWire { + fn from(signal: BoundarySignal) -> Self { + match signal { + BoundarySignal::Term => Self::Term, + BoundarySignal::Kill => Self::Kill, + BoundarySignal::Int => Self::Int, + BoundarySignal::Hup => Self::Hup, + } + } +} + +impl From for BoundarySignal { + fn from(signal: SignalWire) -> Self { + match signal { + SignalWire::Term => Self::Term, + SignalWire::Kill => Self::Kill, + SignalWire::Int => Self::Int, + SignalWire::Hup => Self::Hup, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum ExitStatusWire { + Exited(i32), + Signaled(i32), +} + +impl From for BoundaryExitStatus { + fn from(status: ExitStatusWire) -> Self { + match status { + ExitStatusWire::Exited(code) => Self::Exited(code), + ExitStatusWire::Signaled(signal) => Self::Signaled(signal), + } + } +} + +impl From for ExitStatusWire { + fn from(status: BoundaryExitStatus) -> Self { + match status { + BoundaryExitStatus::Exited(code) => Self::Exited(code), + BoundaryExitStatus::Signaled(signal) => Self::Signaled(signal), + } + } +} + +pub fn encode_frame(message: &T) -> Result, FrameError> { + let payload = serde_json::to_vec(message).map_err(FrameError::Serialize)?; + if payload.len() > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(payload.len())); + } + let length = u32::try_from(payload.len()).map_err(|_| FrameError::TooLarge(payload.len()))?; + let mut frame = Vec::with_capacity(4 + payload.len()); + frame.extend_from_slice(&length.to_be_bytes()); + frame.extend_from_slice(&payload); + Ok(frame) +} + +pub fn decode_frame(frame: &[u8]) -> Result { + let header: [u8; 4] = frame + .get(..4) + .ok_or(FrameError::Truncated)? + .try_into() + .map_err(|_| FrameError::Truncated)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let payload = frame.get(4..).ok_or(FrameError::Truncated)?; + if payload.len() != declared { + return Err(FrameError::LengthMismatch { + declared, + actual: payload.len(), + }); + } + serde_json::from_slice(payload).map_err(FrameError::Deserialize) +} + +pub fn read_frame(reader: &mut impl Read) -> Result { + let mut header = [0_u8; 4]; + reader.read_exact(&mut header)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + reader.read_exact(&mut frame[4..])?; + decode_frame(&frame) +} + +pub fn write_frame(writer: &mut impl Write, message: &T) -> Result<(), FrameError> { + let frame = encode_frame(message)?; + writer.write_all(&frame)?; + writer.flush()?; + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum FrameError { + #[error("control frame is truncated")] + Truncated, + #[error("control frame is too large: {0} bytes")] + TooLarge(usize), + #[error("control frame declared {declared} bytes but contained {actual}")] + LengthMismatch { declared: usize, actual: usize }, + #[error("serialize control frame: {0}")] + Serialize(serde_json::Error), + #[error("deserialize control frame: {0}")] + Deserialize(serde_json::Error), + #[error("read or write control frame: {0}")] + Io(#[from] io::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_round_trips_and_redacts_token() { + let request = RequestEnvelope { + request_id: 7, + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "never-log-this".to_string(), + request: Request::StartAgent { + sandbox_id: "sandbox-1".to_string(), + spec: AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + policy: Box::new(SandboxPolicyWire::from(SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + })), + ca_cert: Some(b"test certificate".to_vec()), + ca_bundle: Some(b"test bundle".to_vec()), + provider_env: std::collections::HashMap::from([( + "OPENAI_API_KEY".to_string(), + "test credential".to_string(), + )]), + }, + }; + let frame = encode_frame(&request).expect("encode request"); + let decoded: RequestEnvelope = decode_frame(&frame).expect("decode request"); + assert_eq!(decoded, request); + let debug = format!("{request:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + assert!(!debug.contains("test credential")); + assert!(!debug.contains("test certificate")); + assert!(!debug.contains("test bundle")); + assert!(debug.contains("OPENAI_API_KEY")); + } + + #[test] + fn rejects_declared_oversize() { + let oversized = u32::try_from(MAX_CONTROL_FRAME_BYTES + 1).expect("test size fits u32"); + let mut frame = Vec::from(oversized.to_be_bytes()); + frame.extend_from_slice(b"{}"); + assert!(matches!( + decode_frame::(&frame), + Err(FrameError::TooLarge(_)) + )); + } + + #[test] + fn resource_claims_reject_empty_or_ambiguous_identities() { + assert!( + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + String::new() + ),])) + .is_err() + ); + assert!( + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod uid".to_string(), + "uid-1".to_string() + ),])) + .is_err() + ); + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + "uid-1".to_string(), + )])) + .expect("opaque resource identity should be valid"); + } +} diff --git a/crates/openshell-isolation-interface/src/lib.rs b/crates/openshell-isolation-interface/src/lib.rs index d7f4e32183..8ec9d96e58 100644 --- a/crates/openshell-isolation-interface/src/lib.rs +++ b/crates/openshell-isolation-interface/src/lib.rs @@ -47,4 +47,8 @@ pub struct AgentSpec { pub interactive: bool, } +/// Versioned control-to-boundary wire types shared by every backend. +pub mod boundary_protocol; pub mod contract; +/// Reusable control-side implementation for a remote boundary endpoint. +pub mod remote; diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs new file mode 100644 index 0000000000..0ea35aafcb --- /dev/null +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -0,0 +1,944 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Host-side RFC 0012 backend for an already-provisioned remote boundary. + +#![allow(unsafe_code)] + +use std::mem::size_of; +use std::os::fd::{FromRawFd as _, IntoRawFd as _}; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use crate::AgentSpec; +use crate::contract::{ + BackendError, BoundBoundary, BoundaryDuplexStream, BoundaryExec, BoundaryExitStatus, + BoundaryInput, BoundaryOutput, BoundaryPortForward, BoundaryProcess, BoundarySignal, + BoundaryTerminal, ExecSession, ExecSpec, INTERFACE_VERSION, IsolationBackend, LoopbackTarget, + MediatedConnection, NetworkMediationSource, ReadyBoundary, RunningBoundary, SandboxContext, + VerifiedTopologyDescriptor, +}; +use async_trait::async_trait; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::UnixStream; +use tokio::sync::Notify; + +use crate::boundary_protocol::{ + AgentSpecWire, BOUNDARY_PROTOCOL_VERSION, BoundaryTopology, BoundaryTransport, ExecSpecWire, + ExitStatusWire, MAX_CONTROL_FRAME_BYTES, Request, RequestEnvelope, Response, ResponseEnvelope, + STREAM_EXIT, STREAM_STDERR, STREAM_STDIN, STREAM_STDIN_CLOSED, STREAM_STDOUT, + SandboxPolicyWire, SignalWire, decode_frame, encode_frame, read_stream_frame, + validate_resource_claims, write_stream_frame, +}; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// How long one control call keeps retrying boundary connect attempts. Boot-time +/// callers retry whole calls above this; past boot, exhausting this window +/// means the remote boundary (or its launcher) is gone rather than still starting. +const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +const MIN_BOOTSTRAP_TOKEN_BYTES: usize = 32; + +/// Host-side remote boundary implementation registered with the supervisor. +#[derive(Debug)] +pub struct RemoteIsolationBackend { + backend_name: String, + ca_file_paths: Arc>>, + provider_env: std::collections::HashMap, +} + +impl RemoteIsolationBackend { + pub fn new( + backend_name: impl Into, + ca_file_paths: Arc>>, + provider_env: std::collections::HashMap, + ) -> Self { + Self { + backend_name: backend_name.into(), + ca_file_paths, + provider_env, + } + } +} + +#[async_trait] +impl IsolationBackend for RemoteIsolationBackend { + fn backend_name(&self) -> &str { + &self.backend_name + } + + fn version(&self) -> u32 { + INTERFACE_VERSION + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + let topology: BoundaryTopology = serde_json::from_slice(descriptor.payload()) + .map_err(|error| BackendError::Descriptor(format!("decode topology: {error}")))?; + validate_topology(&topology, &sandbox)?; + let host_gateway_ip = topology.host_gateway_ip; + let resource_claims = topology.resource_claims.clone(); + let client = Arc::new(BoundaryClient::new(topology)); + expect_response( + client + .call_idempotent(Request::Attach { + policy: Box::new(SandboxPolicyWire::from(sandbox.policy.clone())), + resource_claims, + }) + .await?, + "attached", + )?; + Ok(Box::new(RemoteBound { + client: client.clone(), + agent: sandbox.agent, + policy: sandbox.policy, + sandbox_id: sandbox.sandbox_id, + mediation: Arc::new(RemoteNetworkMediation { client }), + host_gateway_ip, + ca_file_paths: self.ca_file_paths.clone(), + provider_env: self.provider_env.clone(), + })) + } +} + +fn validate_topology( + topology: &BoundaryTopology, + sandbox: &SandboxContext, +) -> Result<(), BackendError> { + if topology.protocol_version != BOUNDARY_PROTOCOL_VERSION { + return Err(BackendError::Descriptor(format!( + "boundary protocol version {} unsupported (expected {BOUNDARY_PROTOCOL_VERSION})", + topology.protocol_version + ))); + } + if topology.boundary_id != sandbox.sandbox_id { + return Err(BackendError::Descriptor(format!( + "boundary {:?} does not match sandbox {:?}", + topology.boundary_id, sandbox.sandbox_id + ))); + } + if topology.bootstrap_token.len() < MIN_BOOTSTRAP_TOKEN_BYTES { + return Err(BackendError::Descriptor(format!( + "boundary bootstrap token must be at least {MIN_BOOTSTRAP_TOKEN_BYTES} bytes" + ))); + } + validate_resource_claims(&topology.resource_claims)?; + match &topology.transport { + BoundaryTransport::Unix { socket_path } => validate_socket_path(socket_path)?, + BoundaryTransport::Tcp { address } => { + if address.port() == 0 || address.ip().is_unspecified() { + return Err(BackendError::Descriptor( + "boundary TCP address must have a concrete IP and nonzero port".to_string(), + )); + } + } + BoundaryTransport::Vsock { + guest_cid, + control_port, + } => { + if *guest_cid < 3 { + return Err(BackendError::Descriptor( + "boundary CID must be at least 3".to_string(), + )); + } + validate_control_port(*control_port)?; + } + } + Ok(()) +} + +fn validate_socket_path(path: &std::path::Path) -> Result<(), BackendError> { + if path.is_absolute() { + Ok(()) + } else { + Err(BackendError::Descriptor( + "boundary control Unix socket path must be absolute".to_string(), + )) + } +} + +fn validate_control_port(port: u32) -> Result<(), BackendError> { + if port == 0 { + Err(BackendError::Descriptor( + "boundary control port must be nonzero".to_string(), + )) + } else { + Ok(()) + } +} + +struct RemoteBound { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + mediation: Arc, + host_gateway_ip: Option, + ca_file_paths: Arc>>, + provider_env: std::collections::HashMap, +} + +#[async_trait] +impl BoundBoundary for RemoteBound { + fn network_mediation_source(&self) -> Arc { + self.mediation.clone() + } + + fn host_gateway_ip(&self) -> Option { + self.host_gateway_ip + } + + async fn confirm(self: Box) -> Result, BackendError> { + expect_response( + self.client.call_idempotent(Request::Confirm).await?, + "confirmed", + )?; + Ok(Box::new(RemoteReady { + client: self.client, + agent: self.agent, + policy: self.policy, + sandbox_id: self.sandbox_id, + ca_file_paths: self.ca_file_paths, + provider_env: self.provider_env, + })) + } +} + +struct RemoteReady { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + ca_file_paths: Arc>>, + provider_env: std::collections::HashMap, +} + +#[async_trait] +impl ReadyBoundary for RemoteReady { + async fn start_agent(self: Box) -> Result, BackendError> { + let ca_paths = self + .ca_file_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let (ca_cert, ca_bundle) = if let Some((ca_cert, ca_bundle)) = ca_paths { + let ca_cert = tokio::fs::read(&ca_cert).await.map_err(|error| { + BackendError::Process(format!("read host proxy CA {}: {error}", ca_cert.display())) + })?; + let ca_bundle = tokio::fs::read(&ca_bundle).await.map_err(|error| { + BackendError::Process(format!( + "read host proxy CA bundle {}: {error}", + ca_bundle.display() + )) + })?; + (Some(ca_cert), Some(ca_bundle)) + } else { + (None, None) + }; + let response = self + .client + .call(Request::StartAgent { + sandbox_id: self.sandbox_id, + spec: AgentSpecWire::from(self.agent), + policy: Box::new(SandboxPolicyWire::from(self.policy)), + ca_cert, + ca_bundle, + provider_env: self.provider_env, + }) + .await?; + let Response::Started { process_id } = response else { + return Err(unexpected_response("started", &response)); + }; + let process = Arc::new(RemoteProcess { + client: self.client.clone(), + process_id, + }); + Ok(Box::new(RemoteRunning { + process, + exec: Arc::new(RemoteExec { + client: self.client.clone(), + }), + port_forward: Arc::new(RemotePortForward { + client: self.client, + }), + })) + } +} + +struct RemoteRunning { + process: Arc, + exec: Arc, + port_forward: Arc, +} + +impl RunningBoundary for RemoteRunning { + fn agent(&self) -> Arc { + self.process.clone() + } + + fn exec(&self) -> Arc { + self.exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +struct RemoteProcess { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryProcess for RemoteProcess { + async fn wait(&self) -> Result { + let response = self + .client + .call_wait(Request::Wait { + process_id: self.process_id.clone(), + }) + .await + .map_err(|error| match error { + // A wait that can no longer reach the boundary leaf means the + // boundary is gone, not that a retry could still observe the + // exit status; report boundary loss per the contract. + BackendError::Unavailable(message) => { + BackendError::Terminated(format!("boundary lost during wait: {message}")) + } + error => error, + })?; + let Response::Exited { status } = response else { + return Err(unexpected_response("exited", &response)); + }; + Ok(status.into()) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Signal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?; + expect_response(response, "signaled") + } + + async fn terminate(&self) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Terminate { + process_id: self.process_id.clone(), + }) + .await?; + expect_response(response, "terminated") + } +} + +struct RemoteExec { + client: Arc, +} + +#[async_trait] +impl BoundaryExec for RemoteExec { + async fn exec(&self, spec: ExecSpec) -> Result { + open_exec_session(self.client.clone(), spec).await + } +} + +struct RemotePortForward { + client: Arc, +} + +#[async_trait] +impl BoundaryPortForward for RemotePortForward { + async fn connect(&self, target: LoopbackTarget) -> Result { + let (stream, response) = self + .client + .call_stream(Request::PortForward { + host: target.host(), + port: target.port(), + }) + .await?; + match response { + Response::PortConnected => Ok(stream), + response => Err(unexpected_response("port_connected", &response)), + } + } +} + +struct RemoteExecProcess { + client: Arc, + process_id: String, + exit: Arc, +} + +struct RemoteExit { + result: std::sync::Mutex>>, + changed: Notify, +} + +impl RemoteExit { + fn new() -> Self { + Self { + result: std::sync::Mutex::new(None), + changed: Notify::new(), + } + } + + fn set(&self, result: Result) { + let mut current = self + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current.is_none() { + *current = Some(result); + self.changed.notify_waiters(); + } + } + + async fn wait(&self) -> Result { + loop { + let changed = self.changed.notified(); + let result = self + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + if let Some(result) = result { + return result.map_err(BackendError::Terminated); + } + changed.await; + } + } +} + +#[async_trait] +impl BoundaryProcess for RemoteExecProcess { + async fn wait(&self) -> Result { + self.exit.wait().await + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + expect_response( + self.client + .call(Request::ExecSignal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?, + "signaled", + ) + } + + async fn terminate(&self) -> Result<(), BackendError> { + self.signal(BoundarySignal::Kill).await + } +} + +struct RemoteTerminal { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryTerminal for RemoteTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Resize { + process_id: self.process_id.clone(), + cols, + rows, + }) + .await?; + if matches!(response, Response::Resized) { + Ok(()) + } else { + Err(unexpected_response("resized", &response)) + } + } +} + +async fn open_exec_session( + client: Arc, + spec: ExecSpec, +) -> Result { + let (stream, response) = client + .call_stream(Request::Exec { + spec: ExecSpecWire::from(spec), + }) + .await?; + let Response::ExecStarted { process_id, pty } = response else { + return Err(unexpected_response("exec_started", &response)); + }; + let (network_reader, network_writer) = tokio::io::split(stream); + let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); + let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); + let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); + let exit = Arc::new(RemoteExit::new()); + tokio::spawn(pump_exec_input(stdin_pump, network_writer)); + tokio::spawn(pump_exec_responses( + network_reader, + stdout_pump, + stderr_pump, + exit.clone(), + )); + + let process: Arc = Arc::new(RemoteExecProcess { + client: client.clone(), + process_id: process_id.clone(), + exit, + }); + let terminal: Option> = if pty { + Some(Arc::new(RemoteTerminal { client, process_id })) + } else { + None + }; + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let stderr: Option = if pty { None } else { Some(Box::new(stderr)) }; + Ok(ExecSession { + process, + stdin: Some(stdin), + stdout, + stderr, + terminal, + }) +} + +async fn pump_exec_input( + mut input: tokio::io::DuplexStream, + mut network: tokio::io::WriteHalf, +) { + let mut buffer = vec![0; 16 * 1024]; + loop { + match input.read(&mut buffer).await { + Ok(0) => { + let _ = write_stream_frame(&mut network, STREAM_STDIN_CLOSED, &[]).await; + return; + } + Ok(read) => { + if write_stream_frame(&mut network, STREAM_STDIN, &buffer[..read]) + .await + .is_err() + { + return; + } + } + Err(_) => return, + } + } +} + +async fn pump_exec_responses( + mut network: tokio::io::ReadHalf, + mut stdout: tokio::io::DuplexStream, + mut stderr: tokio::io::DuplexStream, + exit: Arc, +) { + loop { + match read_stream_frame(&mut network).await { + Ok(Some((STREAM_STDOUT, payload))) => { + if stdout.write_all(&payload).await.is_err() { + exit.set(Err("boundary exec stdout consumer closed".to_string())); + return; + } + } + Ok(Some((STREAM_STDERR, payload))) => { + if stderr.write_all(&payload).await.is_err() { + exit.set(Err("boundary exec stderr consumer closed".to_string())); + return; + } + } + Ok(Some((STREAM_EXIT, payload))) => { + let result = serde_json::from_slice::(&payload) + .map(BoundaryExitStatus::from) + .map_err(|error| format!("decode boundary exec exit: {error}")); + exit.set(result); + return; + } + Ok(Some((channel, _))) => { + exit.set(Err(format!( + "boundary exec returned unexpected stream channel {channel}" + ))); + return; + } + Ok(None) => { + exit.set(Err( + "boundary exec stream closed before exit status".to_string() + )); + return; + } + Err(error) => { + exit.set(Err(format!("read boundary exec stream: {error}"))); + return; + } + } + } +} + +/// Pulls boundary proxy connections over one authenticated vsock stream each. +struct RemoteNetworkMediation { + client: Arc, +} + +#[async_trait] +impl NetworkMediationSource for RemoteNetworkMediation { + async fn accept(&self) -> Result { + let (stream, response) = self.client.open_exchange(Request::AcceptNetwork).await?; + let Response::NetworkConnected { identity } = response else { + return Err(unexpected_response("network_connected", &response)); + }; + Ok(MediatedConnection { + stream, + binary_identity: identity.into_result(), + destination: None, + }) + } +} + +struct BoundaryClient { + topology: BoundaryTopology, + next_request_id: AtomicU64, +} + +impl BoundaryClient { + fn new(topology: BoundaryTopology) -> Self { + Self { + topology, + next_request_id: AtomicU64::new(1), + } + } + + async fn call(&self, request: Request) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, self.exchange(request)) + .await + .map_err(|_| { + BackendError::Unavailable("boundary control request timed out".to_string()) + })? + } + + async fn call_idempotent(&self, request: Request) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, async { + loop { + match self.exchange(request.clone()).await { + Ok(response) => return Ok(response), + Err(BackendError::Unavailable(_)) => { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(error) => return Err(error), + } + } + }) + .await + .map_err(|_| { + BackendError::Unavailable( + "boundary idempotent control request timed out while waiting for remote boundary boot".to_string(), + ) + })? + } + + async fn call_wait(&self, request: Request) -> Result { + self.exchange(request).await + } + + async fn call_stream( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + tokio::time::timeout(REQUEST_TIMEOUT, self.open_exchange(request)) + .await + .map_err(|_| { + BackendError::Unavailable("boundary stream request timed out".to_string()) + })? + } + + async fn exchange(&self, request: Request) -> Result { + let (_, response) = self.open_exchange(request).await?; + Ok(response) + } + + async fn open_exchange( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + let request_id = self.next_request_id.fetch_add(1, Ordering::Relaxed); + let envelope = RequestEnvelope { + request_id, + boundary_id: self.topology.boundary_id.clone(), + bootstrap_token: self.topology.bootstrap_token.clone(), + request, + }; + let mut stream = self.connect_boundary().await?; + let frame = encode_frame(&envelope) + .map_err(|error| BackendError::Process(format!("encode control request: {error}")))?; + stream.write_all(&frame).await.map_err(|error| { + BackendError::Unavailable(format!("write boundary control request: {error}")) + })?; + let mut header = [0_u8; 4]; + stream.read_exact(&mut header).await.map_err(|error| { + BackendError::Unavailable(format!("read boundary control response header: {error}")) + })?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(BackendError::Process(format!( + "boundary control response is too large: {declared} bytes" + ))); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + stream.read_exact(&mut frame[4..]).await.map_err(|error| { + BackendError::Unavailable(format!("read boundary control response: {error}")) + })?; + let response: ResponseEnvelope = decode_frame(&frame) + .map_err(|error| BackendError::Process(format!("decode control response: {error}")))?; + if response.request_id != request_id { + return Err(BackendError::Process(format!( + "boundary response ID {} did not match request ID {request_id}", + response.request_id + ))); + } + let response = match response.response { + Response::Error { kind, message } => Err(guest_error(&kind, message)), + response => Ok(response), + }?; + Ok((stream, response)) + } + + async fn connect_boundary(&self) -> Result { + let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; + loop { + match self.connect_boundary_once().await { + Ok(stream) => return Ok(stream), + Err(error) if tokio::time::Instant::now() >= deadline => return Err(error), + Err(_) => tokio::time::sleep(Duration::from_millis(25)).await, + } + } + } + + async fn connect_boundary_once(&self) -> Result { + match &self.topology.transport { + BoundaryTransport::Unix { socket_path } => { + let stream = UnixStream::connect(socket_path).await.map_err(|error| { + BackendError::Unavailable(format!( + "connect to mapped boundary control socket {}: {error}", + socket_path.display() + )) + })?; + Ok(Box::new(stream)) + } + BoundaryTransport::Tcp { address } => { + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[*address]) + .await + .map_err(|error| { + BackendError::Unavailable(format!( + "connect to boundary TCP endpoint {address}: {error}" + )) + })?; + Ok(Box::new(stream)) + } + BoundaryTransport::Vsock { + guest_cid, + control_port, + } => connect_host_vsock(*guest_cid, *control_port), + } + } +} + +#[cfg(target_os = "linux")] +fn connect_host_vsock( + guest_cid: u32, + control_port: u32, +) -> Result { + let fd = unsafe { libc::socket(libc::AF_VSOCK, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) }; + if fd < 0 { + return Err(BackendError::Unavailable(format!( + "create host vsock: {}", + std::io::Error::last_os_error() + ))); + } + let fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(fd) }; + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|error| { + BackendError::Unavailable(format!("convert host vsock address family: {error}")) + })?; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: control_port, + svm_cid: guest_cid, + svm_zero: [0; 4], + }; + let address_length = + libc::socklen_t::try_from(size_of::()).map_err(|error| { + BackendError::Unavailable(format!("convert host vsock address length: {error}")) + })?; + let result = unsafe { + libc::connect( + std::os::fd::AsRawFd::as_raw_fd(&fd), + (&raw const address).cast::(), + address_length, + ) + }; + if result != 0 { + return Err(BackendError::Unavailable(format!( + "connect host vsock CID {guest_cid} port {control_port}: {}", + std::io::Error::last_os_error() + ))); + } + let stream = unsafe { std::os::unix::net::UnixStream::from_raw_fd(fd.into_raw_fd()) }; + stream.set_nonblocking(true).map_err(|error| { + BackendError::Unavailable(format!("set host vsock nonblocking: {error}")) + })?; + let stream = UnixStream::from_std(stream).map_err(|error| { + BackendError::Unavailable(format!("register host vsock with Tokio: {error}")) + })?; + Ok(Box::new(stream)) +} + +#[cfg(not(target_os = "linux"))] +fn connect_host_vsock( + _guest_cid: u32, + _control_port: u32, +) -> Result { + Err(BackendError::Unavailable( + "host AF_VSOCK transport is supported only on Linux".to_string(), + )) +} + +fn expect_response(response: Response, expected: &str) -> Result<(), BackendError> { + let matches = matches!( + (&response, expected), + (Response::Attached, "attached") + | (Response::Confirmed, "confirmed") + | (Response::Signaled, "signaled") + | (Response::Terminated, "terminated") + ); + if matches { + Ok(()) + } else { + Err(unexpected_response(expected, &response)) + } +} + +fn unexpected_response(expected: &str, response: &Response) -> BackendError { + BackendError::Process(format!( + "expected boundary response {expected:?}, received {response:?}" + )) +} + +fn guest_error(kind: &str, message: String) -> BackendError { + let message = format!("boundary process leaf: {message}"); + match kind { + "invalid" => BackendError::Descriptor(message), + "denied" => BackendError::Denied(message), + "unavailable" => BackendError::Unavailable(message), + "terminated" => BackendError::Terminated(message), + _ => BackendError::Process(message), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + + use super::*; + + fn sandbox() -> SandboxContext { + SandboxContext { + sandbox_id: "sandbox-1".to_string(), + policy: SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }, + agent: AgentSpec { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + } + } + + #[test] + fn topology_debug_redacts_token() { + let topology = BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from("/tmp/vsock.sock"), + }, + host_gateway_ip: Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + resource_claims: std::collections::BTreeMap::new(), + bootstrap_token: "never-log-this-never-log-this".to_string(), + }; + let debug = format!("{topology:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn topology_must_match_sandbox() { + let topology = BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "other".to_string(), + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from("/tmp/vsock.sock"), + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + assert!(matches!( + validate_topology(&topology, &sandbox()), + Err(BackendError::Descriptor(_)) + )); + } + + #[test] + fn topology_rejects_an_unspecified_tcp_target() { + let topology = BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + transport: BoundaryTransport::Tcp { + address: "0.0.0.0:5500".parse().expect("valid address"), + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + assert!(matches!( + validate_topology(&topology, &sandbox()), + Err(BackendError::Descriptor(_)) + )); + } + + #[test] + fn topology_accepts_a_concrete_tcp_target() { + let topology = BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + transport: BoundaryTransport::Tcp { + address: "10.42.0.7:5500".parse().expect("valid address"), + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + validate_topology(&topology, &sandbox()).expect("TCP topology should be valid"); + } +} diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 3463f03767..d9636b2eba 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-extension-core = { path = "../openshell-extension-core" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } @@ -44,6 +45,7 @@ nix = { workspace = true } rustls = { workspace = true } # Serialization (serde_json::json! for OCSF unmapped fields) +base64 = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } prost = { workspace = true } @@ -54,6 +56,9 @@ uuid = { workspace = true } tracing-subscriber = { workspace = true } tracing-appender = { workspace = true } +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2" + [features] default = ["telemetry", "bundled-ca-roots"] ## Convenience alias: all defaults except bundled CA roots. Use diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs new file mode 100644 index 0000000000..6574f4fc06 --- /dev/null +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -0,0 +1,1338 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared implementation of `openshell-sandbox --mode=boundary`. +//! +//! This is transport and lifecycle glue, not another supervisor model. When +//! the control role authorizes `start_agent`, it invokes the existing process +//! supervisor inside the driver-provisioned boundary. + +#![allow(unsafe_code)] + +use std::path::Path; + +#[cfg(target_os = "linux")] +mod linux { + use super::Path; + use std::ffi::CString; + use std::fs::File; + use std::io::{self, Read, Write}; + use std::mem::size_of; + use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd as _, OwnedFd}; + use std::os::unix::ffi::OsStrExt as _; + use std::os::unix::fs::PermissionsExt as _; + use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; + use std::sync::{Arc, Condvar, Mutex}; + use std::time::Duration; + + use openshell_core::proposals::AgentProposals; + use openshell_core::provider_credentials::ProviderCredentialState; + use openshell_isolation_interface::contract::{ + BoundaryExec, BoundaryPortForward, BoundaryProcess, BoundaryTerminal, ExecSession, + LoopbackTarget, + }; + use openshell_supervisor_network::identity_source::ProcfsIdentityResolver; + use openshell_supervisor_process::boundary_io::BoundaryRuntimeState; + use openshell_supervisor_process::delegated::{AgentSignaler, spawn_workload}; + use openshell_supervisor_process::identity::{DriverIdentity, resolve_process_identity}; + use openshell_supervisor_process::netns::{ + NetworkNamespace, create_conformant_netns_for_proxy, + }; + use openshell_supervisor_process::process::{ + ProcessEnforcementMode, ProcessStatus, ResolvedProcessIdentity, + }; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + use openshell_isolation_interface::boundary_protocol::{ + AgentSpecWire, BOUNDARY_PROTOCOL_VERSION, BinaryIdentityWire, BoundaryAgentIdentity, + BoundaryConfig, BoundaryListener as BoundaryListenerConfig, ExecSpecWire, ExitStatusWire, + Request, RequestEnvelope, Response, ResponseEnvelope, STREAM_EXIT, STREAM_STDERR, + STREAM_STDIN, STREAM_STDIN_CLOSED, STREAM_STDOUT, SandboxPolicyWire, SignalWire, + read_frame, read_stream_frame, validate_resource_claims, write_frame, write_stream_frame, + }; + + const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30); + + pub fn run_boundary(config_path: &Path) -> Result<(), String> { + if std::process::id() == 1 { + prepare_pid1_filesystems()?; + } + let bytes = std::fs::read(config_path) + .map_err(|error| format!("read boundary config {}: {error}", config_path.display()))?; + let config: BoundaryConfig = serde_json::from_slice(&bytes).map_err(|error| { + format!("decode boundary config {}: {error}", config_path.display()) + })?; + validate_config(&config)?; + if config.protect_config_file { + protect_boundary_config_file(config_path)?; + } + openshell_supervisor_process::netns::configure_trusted_runtime_root( + config.trusted_runtime_root.clone(), + ) + .map_err(|error| format!("configure trusted boundary helper runtime: {error}"))?; + let child_env = serde_json::to_string(&config.child_env) + .map_err(|error| format!("encode boundary workload environment: {error}"))?; + // This runs before the Tokio runtime or control threads exist. The process + // supervisor consumes the serialized map and applies values only to + // workload children. + unsafe { + std::env::set_var(openshell_core::sandbox_env::USER_ENVIRONMENT, child_env); + } + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| format!("create boundary process runtime: {error}"))?; + let runtime = Arc::new(BoundaryRuntime::new( + config.clone(), + process_runtime.handle().clone(), + )); + serve(&config.listener, runtime) + } + + fn protect_boundary_config_file(config_path: &Path) -> Result<(), String> { + let path = CString::new(config_path.as_os_str().as_bytes()).map_err(|error| { + format!( + "boundary config path {} contains NUL: {error}", + config_path.display() + ) + })?; + if unsafe { libc::chown(path.as_ptr(), 0, 0) } != 0 { + return Err(format!( + "re-own boundary config {} as root: {}", + config_path.display(), + io::Error::last_os_error() + )); + } + std::fs::set_permissions(config_path, std::fs::Permissions::from_mode(0o600)).map_err( + |error| { + format!( + "restrict boundary config {} to root: {error}", + config_path.display() + ) + }, + ) + } + + fn validate_config(config: &BoundaryConfig) -> Result<(), String> { + if config.protocol_version != BOUNDARY_PROTOCOL_VERSION { + return Err(format!( + "boundary protocol version {} unsupported (expected {BOUNDARY_PROTOCOL_VERSION})", + config.protocol_version + )); + } + if config.boundary_id.is_empty() { + return Err("boundary ID must not be empty".to_string()); + } + if config.bootstrap_token.len() < 32 { + return Err("boundary bootstrap token must contain at least 32 bytes".to_string()); + } + validate_resource_claims(&config.resource_claims).map_err(|error| error.to_string())?; + match &config.listener { + BoundaryListenerConfig::Unix { socket_path } if !socket_path.is_absolute() => { + return Err("boundary Unix socket path must be absolute".to_string()); + } + BoundaryListenerConfig::Tcp { address } if address.port() == 0 => { + return Err("boundary TCP listener port must be nonzero".to_string()); + } + BoundaryListenerConfig::Vsock { control_port: 0 } => { + return Err("boundary control port must be nonzero".to_string()); + } + BoundaryListenerConfig::Unix { .. } + | BoundaryListenerConfig::Tcp { .. } + | BoundaryListenerConfig::Vsock { .. } => {} + } + match &config.agent_identity { + BoundaryAgentIdentity::Resolved { uid, gid } if *uid == 0 || *gid == 0 => { + return Err("boundary agent UID and GID must be nonzero".to_string()); + } + BoundaryAgentIdentity::Resolved { .. } + | BoundaryAgentIdentity::OciUser { .. } + | BoundaryAgentIdentity::None => {} + } + if !config.trusted_runtime_root.is_absolute() { + return Err("boundary trusted helper runtime root must be absolute".to_string()); + } + Ok(()) + } + + fn prepare_pid1_filesystems() -> Result<(), String> { + for path in ["/proc", "/sys", "/dev", "/run", "/tmp", "/sandbox"] { + std::fs::create_dir_all(path).map_err(|error| format!("create {path}: {error}"))?; + } + mount_if_needed("proc", "/proc", "proc")?; + mount_if_needed("sysfs", "/sys", "sysfs")?; + mount_if_needed("devtmpfs", "/dev", "devtmpfs")?; + std::fs::create_dir_all("/dev/pts").map_err(|error| format!("create /dev/pts: {error}"))?; + mount_if_needed("devpts", "/dev/pts", "devpts")?; + Ok(()) + } + + fn mount_if_needed(source: &str, target: &str, file_system: &str) -> Result<(), String> { + let source = CString::new(source).map_err(|error| error.to_string())?; + let target_c = CString::new(target).map_err(|error| error.to_string())?; + let file_system = CString::new(file_system).map_err(|error| error.to_string())?; + let result = unsafe { + libc::mount( + source.as_ptr(), + target_c.as_ptr(), + file_system.as_ptr(), + 0, + std::ptr::null(), + ) + }; + if result == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::EBUSY) { + Ok(()) + } else { + Err(format!("mount {file_system:?} on {target}: {error}")) + } + } + + fn serve(config: &BoundaryListenerConfig, runtime: Arc) -> Result<(), String> { + let listener = ControlListener::bind(config) + .map_err(|error| format!("bind boundary control listener: {error}"))?; + tracing::info!(?config, "Boundary control listener ready"); + loop { + if runtime.is_shutdown_requested() { + return Ok(()); + } + match listener.accept() { + Ok(stream) => { + let runtime = runtime.clone(); + std::thread::spawn(move || { + if let Err(error) = serve_one(stream, &runtime) { + tracing::warn!(%error, "Boundary control request failed"); + } + }); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(format!("accept boundary control connection: {error}")), + } + } + } + + fn serve_one(mut stream: ControlStream, runtime: &BoundaryRuntime) -> Result<(), String> { + stream + .set_timeout(CONTROL_IO_TIMEOUT) + .map_err(|error| format!("set control timeout: {error}"))?; + let request: RequestEnvelope = + read_frame(&mut stream).map_err(|error| format!("read control frame: {error}"))?; + if !runtime.authenticate(&request) { + let response = ResponseEnvelope { + request_id: request.request_id, + response: guest_error("denied", "control authentication failed"), + }; + return write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}")); + } + match request.request.clone() { + Request::Exec { spec } => { + let (process_id, session) = match runtime.start_exec(spec) { + Ok(started) => started, + Err(response) => { + return write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response, + }, + ) + .map_err(|error| format!("write exec error response: {error}")); + } + }; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::ExecStarted { + process_id: process_id.clone(), + pty: session.terminal.is_some(), + }, + }, + ) + .map_err(|error| format!("write exec start response: {error}"))?; + return runtime.stream_exec(stream, &process_id, session); + } + Request::PortForward { host, port } => { + let target = LoopbackTarget::new(host, port) + .map_err(|error| format!("validate port-forward target: {error}"))?; + let mut target = runtime + .connect_port(target) + .map_err(|error| format!("connect boundary loopback port: {error}"))?; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::PortConnected, + }, + ) + .map_err(|error| format!("write port-forward response: {error}"))?; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + tokio::io::copy_bidirectional(&mut stream, &mut target) + .await + .map_err(|error| format!("bridge boundary loopback stream: {error}")) + })?; + return Ok(()); + } + Request::AcceptNetwork => { + let (mut target, identity) = runtime.accept_network()?; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::NetworkConnected { identity }, + }, + ) + .map_err(|error| format!("write network mediation response: {error}"))?; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + tokio::io::copy_bidirectional(&mut stream, &mut target) + .await + .map_err(|error| { + format!("bridge boundary network mediation stream: {error}") + }) + })?; + return Ok(()); + } + _ => {} + } + let shutdown_after_response = matches!( + &request.request, + Request::Wait { process_id } if runtime.is_main_process(process_id) + ); + let response = ResponseEnvelope { + request_id: request.request_id, + response: runtime.dispatch(request), + }; + write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}"))?; + if shutdown_after_response && matches!(response.response, Response::Exited { .. }) { + runtime.request_shutdown(); + } + Ok(()) + } + + struct BoundaryRuntime { + config: BoundaryConfig, + process_runtime: tokio::runtime::Handle, + state: Mutex, + /// The wire policy bound at first attach, so an idempotent attach retry + /// carrying a different policy is denied instead of silently keeping + /// the first policy. + attached_policy: Mutex>, + next_exec_id: AtomicU64, + exec_handles: Mutex>, + shutdown_requested: AtomicBool, + } + + struct ExecHandle { + process: Arc, + terminal: Option>, + } + + enum RuntimeState { + AwaitingAttach, + Bound(PreparedBoundary), + Ready(PreparedBoundary), + Running(Arc), + } + + #[derive(Clone)] + struct PreparedBoundary { + netns: Option>, + network_listener: Option>, + proxy_port: u16, + } + + async fn bridge_exec_stream( + stream: openshell_isolation_interface::contract::BoundaryDuplexStream, + session: ExecSession, + ) -> Result<(), String> { + let ExecSession { + process, + stdin, + stdout, + stderr, + terminal: _, + } = session; + let (mut network_reader, network_writer) = tokio::io::split(stream); + let network_writer = Arc::new(tokio::sync::Mutex::new(network_writer)); + + let stdin_task = stdin.map(|mut stdin| { + tokio::spawn(async move { + while let Some((channel, payload)) = read_stream_frame(&mut network_reader).await? { + match channel { + STREAM_STDIN => stdin.write_all(&payload).await?, + STREAM_STDIN_CLOSED => { + stdin.shutdown().await?; + break; + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected host-to-boundary stream channel", + )); + } + } + } + Ok::<(), io::Error>(()) + }) + }); + + let stdout_task = tokio::spawn(pump_exec_output( + stdout, + STREAM_STDOUT, + network_writer.clone(), + )); + let stderr_task = stderr.map(|stderr| { + tokio::spawn(pump_exec_output( + stderr, + STREAM_STDERR, + network_writer.clone(), + )) + }); + + let status = process + .wait() + .await + .map_err(|error| format!("wait for boundary exec: {error}"))?; + stdout_task + .await + .map_err(|error| format!("join boundary exec stdout: {error}"))? + .map_err(|error| format!("stream boundary exec stdout: {error}"))?; + if let Some(stderr_task) = stderr_task { + stderr_task + .await + .map_err(|error| format!("join boundary exec stderr: {error}"))? + .map_err(|error| format!("stream boundary exec stderr: {error}"))?; + } + let exit = serde_json::to_vec(&ExitStatusWire::from(status)) + .map_err(|error| format!("encode boundary exec exit: {error}"))?; + write_stream_frame(&mut *network_writer.lock().await, STREAM_EXIT, &exit) + .await + .map_err(|error| format!("write boundary exec exit: {error}"))?; + if let Some(stdin_task) = stdin_task { + stdin_task.abort(); + } + Ok(()) + } + + async fn pump_exec_output( + mut output: openshell_isolation_interface::contract::BoundaryOutput, + channel: u8, + writer: Arc< + tokio::sync::Mutex< + tokio::io::WriteHalf, + >, + >, + ) -> io::Result<()> { + let mut buffer = vec![0; 16 * 1024]; + loop { + let read = output.read(&mut buffer).await?; + if read == 0 { + return Ok(()); + } + write_stream_frame(&mut *writer.lock().await, channel, &buffer[..read]).await?; + } + } + + impl BoundaryRuntime { + fn new(config: BoundaryConfig, process_runtime: tokio::runtime::Handle) -> Self { + Self { + config, + process_runtime, + state: Mutex::new(RuntimeState::AwaitingAttach), + attached_policy: Mutex::new(None), + next_exec_id: AtomicU64::new(1), + exec_handles: Mutex::new(std::collections::HashMap::new()), + shutdown_requested: AtomicBool::new(false), + } + } + + fn is_shutdown_requested(&self) -> bool { + self.shutdown_requested.load(Ordering::Acquire) + } + + fn request_shutdown(&self) { + self.shutdown_requested.store(true, Ordering::Release); + } + + fn is_main_process(&self, process_id: &str) -> bool { + let state = lock(&self.state); + matches!( + &*state, + RuntimeState::Running(process) if process.process_id() == process_id + ) + } + + fn dispatch(&self, envelope: RequestEnvelope) -> Response { + if !self.authenticate(&envelope) { + return guest_error("denied", "control authentication failed"); + } + match envelope.request { + Request::Attach { + policy, + resource_claims, + } => { + if resource_claims == self.config.resource_claims { + self.attach(*policy) + } else { + guest_error( + "denied", + "topology resource claims do not match the boundary configuration", + ) + } + } + Request::Confirm => self.confirm(), + Request::StartAgent { + sandbox_id, + spec, + policy, + ca_cert, + ca_bundle, + provider_env, + } => self.start_agent(sandbox_id, spec, *policy, ca_cert, ca_bundle, provider_env), + Request::Wait { process_id } => self.wait(&process_id), + Request::Signal { process_id, signal } => self.signal(&process_id, signal), + Request::Terminate { process_id } => self.terminate(&process_id), + Request::ExecSignal { process_id, signal } => self.signal_exec(&process_id, signal), + Request::Resize { + process_id, + cols, + rows, + } => self.resize_exec(&process_id, cols, rows), + Request::Exec { .. } | Request::PortForward { .. } | Request::AcceptNetwork => { + guest_error("invalid", "streaming request used on control path") + } + } + } + + fn authenticate(&self, envelope: &RequestEnvelope) -> bool { + constant_time_eq( + envelope.boundary_id.as_bytes(), + self.config.boundary_id.as_bytes(), + ) && constant_time_eq( + envelope.bootstrap_token.as_bytes(), + self.config.bootstrap_token.as_bytes(), + ) + } + + fn start_exec(&self, spec: ExecSpecWire) -> Result<(String, ExecSession), Response> { + let executor = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + process.boundary_exec() + }; + let session = self + .process_runtime + .block_on(executor.exec(spec.into())) + .map_err(|error| guest_error("failed", error.to_string()))?; + let process_id = format!("exec-{}", self.next_exec_id.fetch_add(1, Ordering::Relaxed)); + lock(&self.exec_handles).insert( + process_id.clone(), + ExecHandle { + process: session.process.clone(), + terminal: session.terminal.clone(), + }, + ); + Ok((process_id, session)) + } + + fn signal_exec(&self, process_id: &str, signal: SignalWire) -> Response { + let process = lock(&self.exec_handles) + .get(process_id) + .map(|handle| handle.process.clone()); + let Some(process) = process else { + return guest_error("invalid", "unknown exec process ID"); + }; + match self.process_runtime.block_on(process.signal(signal.into())) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("failed", error.to_string()), + } + } + + fn resize_exec(&self, process_id: &str, cols: u16, rows: u16) -> Response { + let terminal = lock(&self.exec_handles) + .get(process_id) + .and_then(|handle| handle.terminal.clone()); + let Some(terminal) = terminal else { + return guest_error("invalid", "exec process has no terminal"); + }; + match self.process_runtime.block_on(terminal.resize(cols, rows)) { + Ok(()) => Response::Resized, + Err(error) => guest_error("failed", error.to_string()), + } + } + + fn connect_port( + &self, + target: LoopbackTarget, + ) -> Result { + let port_forward = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err("agent process has not been started".to_string()); + }; + process.port_forward() + }; + self.process_runtime + .block_on(port_forward.connect(target)) + .map_err(|error| error.to_string()) + } + + fn accept_network(&self) -> Result<(tokio::net::TcpStream, BinaryIdentityWire), String> { + let process = loop { + let running = { + let state = lock(&self.state); + match &*state { + RuntimeState::Running(process) => Some(process.clone()), + _ => None, + } + }; + if let Some(process) = running { + break process; + } + std::thread::sleep(Duration::from_millis(10)); + }; + let listener = process + .network_listener() + .ok_or_else(|| "network mediation requested for a non-proxy policy".to_string())?; + let (stream, workload_addr) = self + .process_runtime + .block_on(listener.accept()) + .map_err(|error| format!("accept boundary proxy connection: {error}"))?; + let proxy_addr = stream + .local_addr() + .map_err(|error| format!("read boundary proxy address: {error}"))?; + let identity = process + .identity_resolver() + .resolve_connection(workload_addr, proxy_addr); + Ok((stream, BinaryIdentityWire::from(identity))) + } + + fn stream_exec( + &self, + stream: ControlStream, + process_id: &str, + session: ExecSession, + ) -> Result<(), String> { + let process_id = process_id.to_string(); + let bridged = self.process_runtime.block_on(async move { + let stream = stream.into_tokio()?; + bridge_exec_stream(stream, session).await + }); + // Remove the handle even when the bridge fails; a failed stream + // must not leak its exec registration. + lock(&self.exec_handles).remove(&process_id); + bridged + } + + fn attach(&self, policy: SandboxPolicyWire) -> Response { + let mut state = lock(&self.state); + match &*state { + RuntimeState::AwaitingAttach => { + let prepared = match PreparedBoundary::establish( + &self.process_runtime, + &policy.clone().into(), + ) { + Ok(prepared) => prepared, + Err(error) => return guest_error("failed", error), + }; + *lock(&self.attached_policy) = Some(policy); + *state = RuntimeState::Bound(prepared); + Response::Attached + } + RuntimeState::Bound(_) => { + // Idempotent retry of the same attach; a different policy + // must not be silently coalesced onto the bound boundary. + if lock(&self.attached_policy).as_ref() == Some(&policy) { + Response::Attached + } else { + guest_error("denied", "attach policy does not match the bound boundary") + } + } + _ => guest_error("invalid", "boundary has already advanced past attach"), + } + } + + fn confirm(&self) -> Response { + let mut state = lock(&self.state); + match &*state { + RuntimeState::Bound(prepared) => { + if let Err(error) = prepared.confirm(&self.process_runtime) { + return guest_error("failed", error); + } + *state = RuntimeState::Ready(prepared.clone()); + Response::Confirmed + } + RuntimeState::Ready(_) => Response::Confirmed, + RuntimeState::AwaitingAttach => { + guest_error("invalid", "boundary must be attached before confirm") + } + RuntimeState::Running(_) => { + guest_error("invalid", "boundary has already started its agent") + } + } + } + + fn start_agent( + &self, + sandbox_id: String, + spec: AgentSpecWire, + policy: SandboxPolicyWire, + ca_cert: Option>, + ca_bundle: Option>, + provider_env: std::collections::HashMap, + ) -> Response { + let mut state = lock(&self.state); + let RuntimeState::Ready(prepared) = &*state else { + return guest_error("invalid", "boundary must be confirmed before start_agent"); + }; + let ca_file_paths = match install_ca_material(ca_cert, ca_bundle) { + Ok(paths) => paths, + Err(error) => return guest_error("failed", error), + }; + let mut policy = policy.into(); + let driver_identity = match &self.config.agent_identity { + BoundaryAgentIdentity::Resolved { uid, gid } => DriverIdentity::Resolved { + uid: *uid, + gid: *gid, + }, + BoundaryAgentIdentity::OciUser { declaration } => DriverIdentity::OciUser { + declaration: declaration.clone(), + }, + BoundaryAgentIdentity::None => DriverIdentity::None, + }; + let resolved_identity = match resolve_process_identity(&mut policy, &driver_identity) { + Ok(identity) => identity, + Err(error) => return guest_error("failed", error.to_string()), + }; + let launch = ManagedProcessLaunch { + sandbox_id, + spec, + policy, + resolved_identity, + provider_env, + ca_file_paths, + }; + let process = + match ManagedProcess::spawn(&self.process_runtime, launch, prepared.clone()) { + Ok(process) => Arc::new(process), + Err(error) => return guest_error("failed", error), + }; + let process_id = process.process_id(); + *state = RuntimeState::Running(process); + Response::Started { process_id } + } + + fn wait(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.wait() { + Ok(status) => Response::Exited { status }, + Err(error) => guest_error("failed", error), + } + } + + fn signal(&self, process_id: &str, signal: SignalWire) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(signal) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("terminated", error), + } + } + + fn terminate(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(SignalWire::Kill) { + Ok(()) => Response::Terminated, + Err(_) if process.has_exited() => Response::Terminated, + Err(error) => guest_error("failed", error), + } + } + + fn running_process(&self, process_id: &str) -> Result, Response> { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + if process.process_id() != process_id { + return Err(guest_error("invalid", "unknown process ID")); + } + Ok(process.clone()) + } + } + + impl PreparedBoundary { + fn establish( + runtime: &tokio::runtime::Handle, + policy: &openshell_core::policy::SandboxPolicy, + ) -> Result { + let netns = create_conformant_netns_for_proxy(policy) + .map_err(|error| format!("establish boundary workload network namespace: {error}"))? + .map(Arc::new); + let proxy_port = policy + .network + .proxy + .as_ref() + .and_then(|proxy| proxy.http_addr) + .map_or(3128, |address| address.port()); + let network_listener = if let Some(netns) = netns.as_ref() { + let address = std::net::SocketAddr::new(netns.host_ip(), proxy_port); + Some(Arc::new( + runtime + .block_on(tokio::net::TcpListener::bind(address)) + .map_err(|error| { + format!("bind boundary mediation listener {address}: {error}") + })?, + )) + } else { + None + }; + Ok(Self { + netns, + network_listener, + proxy_port, + }) + } + + fn confirm(&self, runtime: &tokio::runtime::Handle) -> Result<(), String> { + if let Some(netns) = self.netns.as_ref() { + runtime + .block_on( + netns + .egress_ceiling_verifier() + .verify_bounded(self.proxy_port, Duration::from_secs(2)), + ) + .map_err(|error| format!("verify boundary egress ceiling: {error}"))?; + if self.network_listener.is_none() { + return Err("boundary proxy namespace has no mediation listener".to_string()); + } + } + Ok(()) + } + } + + fn install_ca_material( + ca_cert: Option>, + ca_bundle: Option>, + ) -> Result, String> { + let (ca_cert, ca_bundle) = match (ca_cert, ca_bundle) { + (Some(ca_cert), Some(ca_bundle)) => (ca_cert, ca_bundle), + (None, None) => return Ok(None), + _ => { + return Err( + "boundary proxy CA certificate and bundle must be supplied together" + .to_string(), + ); + } + }; + let directory = std::path::PathBuf::from("/run/openshell/proxy-ca"); + std::fs::create_dir_all(&directory) + .map_err(|error| format!("create boundary proxy CA directory: {error}"))?; + let ca_path = directory.join("ca.crt"); + let bundle_path = directory.join("ca-bundle.crt"); + std::fs::write(&ca_path, ca_cert) + .map_err(|error| format!("write boundary proxy CA: {error}"))?; + std::fs::write(&bundle_path, ca_bundle) + .map_err(|error| format!("write boundary proxy CA bundle: {error}"))?; + Ok(Some((ca_path, bundle_path))) + } + + type ProcessExit = Result; + type SharedProcessExit = Arc<(Mutex>, Condvar)>; + + struct ManagedProcess { + pid: i32, + signaler: AgentSignaler, + exit: SharedProcessExit, + boundary_exec: Arc, + port_forward: Arc, + network_listener: Option>, + identity_resolver: ProcfsIdentityResolver, + _netns: Option>, + } + + struct ManagedProcessLaunch { + sandbox_id: String, + spec: AgentSpecWire, + policy: openshell_core::policy::SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + } + + impl ManagedProcess { + fn spawn( + runtime: &tokio::runtime::Handle, + launch: ManagedProcessLaunch, + prepared: PreparedBoundary, + ) -> Result { + let ManagedProcessLaunch { + sandbox_id, + spec, + policy, + resolved_identity, + provider_env, + ca_file_paths, + } = launch; + if spec.program.is_empty() { + return Err("agent program must not be empty".to_string()); + } + let boundary_runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); + let entrypoint_pid = Arc::new(AtomicU32::new(0)); + let mut spawned = runtime + .block_on(spawn_workload( + &spec.program, + &spec.args, + spec.workdir.as_deref(), + spec.timeout_secs, + spec.interactive, + Some(&sandbox_id), + None, + None, + false, + &policy, + resolved_identity, + ProcessEnforcementMode::Full, + entrypoint_pid.clone(), + None, + ProviderCredentialState::from_child_env_snapshot(0, provider_env.clone()), + provider_env, + ca_file_paths, + AgentProposals::default(), + prepared.netns.as_deref(), + None, + None, + Some(boundary_runtime), + None, + )) + .map_err(|error| format!("start process supervisor leaf: {error:?}"))?; + let pid = i32::try_from(spawned.pid()) + .map_err(|_| "process supervisor PID does not fit i32".to_string())?; + let signaler = spawned.signaler(); + let boundary_exec = spawned.boundary_exec(); + let port_forward = spawned.port_forward(); + let network_listener = prepared.network_listener.clone(); + let exit = Arc::new((Mutex::new(None), Condvar::new())); + let reaper_exit = exit.clone(); + runtime.spawn(async move { + let result = spawned + .wait() + .await + .map(process_status) + .map_err(|error| format!("wait for process supervisor leaf: {error}")); + let (state, changed) = &*reaper_exit; + *lock(state) = Some(result); + changed.notify_all(); + }); + Ok(Self { + pid, + signaler, + exit, + boundary_exec, + port_forward, + network_listener, + identity_resolver: ProcfsIdentityResolver { entrypoint_pid }, + _netns: prepared.netns, + }) + } + + fn process_id(&self) -> String { + self.pid.to_string() + } + + fn wait(&self) -> ProcessExit { + let (state, changed) = &*self.exit; + let mut exit = lock(state); + while exit.is_none() { + exit = changed + .wait(exit) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + exit.as_ref().expect("exit checked above").clone() + } + + fn signal(&self, signal: SignalWire) -> Result<(), String> { + if self.has_exited() { + return Err("agent process has already exited".to_string()); + } + let result = match signal { + SignalWire::Term => self.signaler.term(), + SignalWire::Kill => self.signaler.kill(), + SignalWire::Int => self.signaler.interrupt(), + SignalWire::Hup => self.signaler.hangup(), + }; + result.map_err(|error| format!("signal process supervisor group: {error}")) + } + + fn has_exited(&self) -> bool { + let (state, _) = &*self.exit; + lock(state).is_some() + } + + fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } + + fn network_listener(&self) -> Option> { + self.network_listener.clone() + } + + fn identity_resolver(&self) -> ProcfsIdentityResolver { + self.identity_resolver.clone() + } + } + + fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn process_status(status: ProcessStatus) -> ExitStatusWire { + status.signal().map_or_else( + || ExitStatusWire::Exited(status.code()), + ExitStatusWire::Signaled, + ) + } + + fn guest_error(kind: &str, message: impl Into) -> Response { + Response::Error { + kind: kind.to_string(), + message: message.into(), + } + } + + fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + let max_len = left.len().max(right.len()); + let mut difference = left.len() ^ right.len(); + for index in 0..max_len { + let left_byte = left.get(index).copied().unwrap_or_default(); + let right_byte = right.get(index).copied().unwrap_or_default(); + difference |= usize::from(left_byte ^ right_byte); + } + difference == 0 + } + + enum ControlListener { + Vsock(OwnedFd), + Unix(std::os::unix::net::UnixListener), + Tcp(std::net::TcpListener), + } + + impl ControlListener { + fn bind(config: &BoundaryListenerConfig) -> io::Result { + match config { + BoundaryListenerConfig::Vsock { control_port } => { + Self::bind_vsock(*control_port).map(Self::Vsock) + } + BoundaryListenerConfig::Unix { socket_path } => { + let listener = std::os::unix::net::UnixListener::bind(socket_path)?; + // The driver-owned parent directory limits discovery and + // reachability; the protocol bootstrap token authenticates + // every request. Cross-UID placements such as rootful Docker + // need write permission on the socket inode itself. + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o666))?; + listener.set_nonblocking(true)?; + Ok(Self::Unix(listener)) + } + BoundaryListenerConfig::Tcp { address } => { + let listener = std::net::TcpListener::bind(address)?; + listener.set_nonblocking(true)?; + Ok(Self::Tcp(listener)) + } + } + } + + fn bind_vsock(port: u32) -> io::Result { + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "AF_VSOCK exceeds sa_family_t") + })?; + let address_length = libc::socklen_t::try_from(size_of::()) + .map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "sockaddr_vm exceeds socklen_t") + })?; + let raw_fd = unsafe { + libc::socket( + libc::AF_VSOCK, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK, + 0, + ) + }; + if raw_fd < 0 { + return Err(io::Error::last_os_error()); + } + let fd = unsafe { OwnedFd::from_raw_fd(raw_fd) }; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: port, + svm_cid: libc::VMADDR_CID_ANY, + svm_zero: [0; 4], + }; + let result = unsafe { + libc::bind( + fd.as_raw_fd(), + (&raw const address).cast::(), + address_length, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + if unsafe { libc::listen(fd.as_raw_fd(), 16) } < 0 { + return Err(io::Error::last_os_error()); + } + Ok(fd) + } + + fn accept(&self) -> io::Result { + match self { + Self::Vsock(fd) => { + let raw_fd = unsafe { + libc::accept4( + fd.as_raw_fd(), + std::ptr::null_mut(), + std::ptr::null_mut(), + libc::SOCK_CLOEXEC, + ) + }; + if raw_fd < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(ControlStream::Vsock(unsafe { File::from_raw_fd(raw_fd) })) + } + } + Self::Unix(listener) => { + let (stream, _) = listener.accept()?; + Ok(ControlStream::Unix(stream)) + } + Self::Tcp(listener) => { + let (stream, _) = listener.accept()?; + if let Err(error) = stream.set_nodelay(true) { + tracing::debug!(%error, "Failed to set boundary TCP_NODELAY"); + } + Ok(ControlStream::Tcp(stream)) + } + } + } + } + + enum ControlStream { + Vsock(File), + Unix(std::os::unix::net::UnixStream), + Tcp(std::net::TcpStream), + } + + impl ControlStream { + fn set_timeout(&self, timeout: Duration) -> io::Result<()> { + if let Self::Unix(stream) = self { + stream.set_read_timeout(Some(timeout))?; + return stream.set_write_timeout(Some(timeout)); + } + if let Self::Tcp(stream) = self { + stream.set_read_timeout(Some(timeout))?; + return stream.set_write_timeout(Some(timeout)); + } + let Self::Vsock(file) = self else { + unreachable!("Unix and TCP streams returned above") + }; + let option_length = + libc::socklen_t::try_from(size_of::()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "timeval exceeds socklen_t") + })?; + let timeout_seconds = timeout.as_secs().try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "control timeout exceeds time_t", + ) + })?; + let timeout = libc::timeval { + tv_sec: timeout_seconds, + tv_usec: timeout.subsec_micros().into(), + }; + for option in [libc::SO_RCVTIMEO, libc::SO_SNDTIMEO] { + let result = unsafe { + libc::setsockopt( + file.as_raw_fd(), + libc::SOL_SOCKET, + option, + (&raw const timeout).cast(), + option_length, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) + } + + fn into_tokio( + self, + ) -> Result { + match self { + Self::Unix(stream) => { + stream.set_nonblocking(true).map_err(|error| { + format!("set boundary Unix stream nonblocking: {error}") + })?; + let stream = tokio::net::UnixStream::from_std(stream).map_err(|error| { + format!("register boundary Unix stream with Tokio: {error}") + })?; + Ok(Box::new(stream)) + } + Self::Tcp(stream) => { + stream + .set_nonblocking(true) + .map_err(|error| format!("set boundary TCP stream nonblocking: {error}"))?; + let stream = tokio::net::TcpStream::from_std(stream).map_err(|error| { + format!("register boundary TCP stream with Tokio: {error}") + })?; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + Ok(Box::new(stream)) + } + Self::Vsock(file) => { + let stream = + unsafe { std::os::unix::net::UnixStream::from_raw_fd(file.into_raw_fd()) }; + stream.set_nonblocking(true).map_err(|error| { + format!("set boundary vsock stream nonblocking: {error}") + })?; + let stream = tokio::net::UnixStream::from_std(stream).map_err(|error| { + format!("register boundary vsock stream with Tokio: {error}") + })?; + Ok(Box::new(stream)) + } + } + } + } + + impl Read for ControlStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self { + Self::Vsock(stream) => stream.read(buffer), + Self::Unix(stream) => stream.read(buffer), + Self::Tcp(stream) => stream.read(buffer), + } + } + } + + impl Write for ControlStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + match self { + Self::Vsock(stream) => stream.write(buffer), + Self::Unix(stream) => stream.write(buffer), + Self::Tcp(stream) => stream.write(buffer), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Vsock(stream) => stream.flush(), + Self::Unix(stream) => stream.flush(), + Self::Tcp(stream) => stream.flush(), + } + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn boundary_config_debug_redacts_token() { + let config = BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "never-log-this-never-log-this".to_string(), + protect_config_file: false, + listener: BoundaryListenerConfig::Vsock { control_port: 5500 }, + resource_claims: std::collections::BTreeMap::new(), + agent_identity: BoundaryAgentIdentity::Resolved { + uid: 10_001, + gid: 10_001, + }, + trusted_runtime_root: std::path::PathBuf::from( + "/opt/openshell/bin/openshell-runtime", + ), + child_env: std::collections::HashMap::new(), + }; + let debug = format!("{config:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn constant_time_comparison_checks_length_and_content() { + assert!(constant_time_eq(b"same", b"same")); + assert!(!constant_time_eq(b"same", b"different")); + assert!(!constant_time_eq(b"same", b"sam")); + } + + #[test] + fn unix_listener_allows_authenticated_cross_uid_control() { + use std::os::unix::fs::PermissionsExt as _; + + let directory = tempfile::tempdir().expect("temporary directory"); + let socket_path = directory.path().join("control.sock"); + let _listener = ControlListener::bind(&BoundaryListenerConfig::Unix { + socket_path: socket_path.clone(), + }) + .expect("bind Unix listener"); + let mode = socket_path + .metadata() + .expect("socket metadata") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o666); + } + + #[test] + fn empty_oci_user_is_resolved_with_the_policy_inside_the_boundary() { + let config = BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "a".repeat(64), + protect_config_file: false, + listener: BoundaryListenerConfig::Vsock { control_port: 5500 }, + resource_claims: std::collections::BTreeMap::new(), + agent_identity: BoundaryAgentIdentity::OciUser { + declaration: String::new(), + }, + trusted_runtime_root: std::path::PathBuf::from("/opt/openshell/runtime"), + child_env: std::collections::HashMap::new(), + }; + + validate_config(&config).unwrap(); + } + } +} + +#[cfg(target_os = "linux")] +pub use linux::run_boundary; + +#[cfg(not(target_os = "linux"))] +pub fn run_boundary(_config_path: &Path) -> Result<(), String> { + Err("boundary mode is supported only on Linux".to_string()) +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index b2ba533515..99aa97c954 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -16,6 +16,7 @@ compile_error!( ); mod activity_aggregator; +mod boundary_server; mod denial_aggregator; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod google_cloud_metadata; @@ -68,6 +69,16 @@ use openshell_ocsf::{ /// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; +/// Run the authenticated boundary-local half of the supervisor. +/// +/// # Errors +/// +/// Returns an error when the protected boundary configuration is invalid or +/// the boundary listener cannot be established. +pub fn run_boundary(config_path: &std::path::Path) -> Result<()> { + boundary_server::run_boundary(config_path).map_err(|error| miette::miette!(error)) +} + use openshell_core::denial::DenialEvent; use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; use openshell_core::proposals::AgentProposals; @@ -127,6 +138,8 @@ pub async fn run_sandbox( network_enabled: bool, process_enabled: bool, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + topology_descriptor: Option, + admitted_isolation_backend: Option, ) -> Result { let (program, args) = command .split_first() @@ -221,9 +234,16 @@ pub async fn run_sandbox( // Normalize the active driver's identity contract once, while both the // policy and launched image filesystem are available. Kubernetes and // OpenShift retain their authoritative numeric pair; Docker fills only - // omitted policy fields from OCI Config.User. + // omitted policy fields from OCI Config.User. A remote boundary resolves + // identity in its own filesystem instead; control must not interpret + // guest account data against the host's /etc/passwd and /etc/group. #[cfg(unix)] - let (resolved_process_identity, workspace) = { + let (resolved_process_identity, workspace) = if topology_descriptor.is_some() { + ( + openshell_supervisor_process::process::ResolvedProcessIdentity::default(), + openshell_supervisor_process::process::ResolvedWorkspace::new(workdir.clone(), false), + ) + } else { let driver_identity = openshell_supervisor_process::identity::DriverIdentity::from_env()?; let use_workdir_as_home = matches!( &driver_identity, @@ -395,24 +415,78 @@ pub async fn run_sandbox( // the entrypoint process's /proc/net/tcp for identity binding. let entrypoint_pid = Arc::new(AtomicU32::new(0)); + // A separated topology uses the shared authenticated boundary protocol. + // The admitted backend name is resolved independently of the protected + // descriptor, and generic supervisor code never imports a driver crate. + let remote_boundary = if let Some(descriptor) = topology_descriptor { + if sidecar_network_enforcement || !network_enabled || !process_enabled { + return Err(miette::miette!( + "--mode=control requires combined network and process supervision" + )); + } + let admitted_backend_name = admitted_isolation_backend.ok_or_else(|| { + miette::miette!("topology descriptor supplied without an admitted isolation backend") + })?; + let ca_file_paths = Arc::new(std::sync::Mutex::new(None)); + let backend: Arc = Arc::new( + openshell_isolation_interface::remote::RemoteIsolationBackend::new( + admitted_backend_name.clone(), + ca_file_paths.clone(), + provider_env.clone(), + ), + ); + let mut registry = openshell_isolation_interface::contract::BackendRegistry::new(); + registry + .register(backend) + .map_err(|error| miette::miette!(error.to_string()))?; + let (backend, verified) = registry + .resolve(descriptor, &admitted_backend_name) + .map_err(|error| miette::miette!(error.to_string()))?; + let context = openshell_isolation_interface::contract::SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + policy: policy.clone(), + agent: openshell_isolation_interface::AgentSpec { + program: program.clone(), + args: args.to_vec(), + workdir: workspace.owned_root(), + timeout_secs, + interactive, + }, + }; + let bound = backend + .attach(verified, context) + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %admitted_backend_name, "Isolation boundary attached"); + Some((bound, admitted_backend_name, ca_file_paths)) + } else { + if admitted_isolation_backend.is_some() { + return Err(miette::miette!( + "admitted isolation backend supplied without a topology descriptor" + )); + } + None + }; + // Create the workload's network namespace. It is shared infrastructure: // the proxy binds to its host-side veth IP, the bypass monitor reads // /dev/kmsg from inside it, and the workload child / SSH sessions enter // it via setns(). The RAII handle lives in this frame for the duration // of the sandbox. #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { + let netns = if remote_boundary.is_none() && network_enabled && !sidecar_network_enforcement { openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? } else { None }; #[cfg(target_os = "linux")] - let transparent_tcp_requested = opa_engine - .as_ref() - .map(|engine| engine.policy_dns_eligibility_snapshot()) - .transpose()? - .is_some_and(|snapshot| !snapshot.endpoints.is_empty()); + let transparent_tcp_requested = remote_boundary.is_none() + && opa_engine + .as_ref() + .map(|engine| engine.policy_dns_eligibility_snapshot()) + .transpose()? + .is_some_and(|snapshot| !snapshot.endpoints.is_empty()); #[cfg(target_os = "linux")] let runtime_capabilities = std::env::var(openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES).ok(); @@ -527,6 +601,16 @@ pub async fn run_sandbox( // API read the current value so proposals target the correct workspace. let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); + let remote_network_source = remote_boundary + .as_ref() + .map(|(bound, _, _)| bound.network_mediation_source()); + let remote_dns_source = remote_boundary + .as_ref() + .and_then(|(bound, _, _)| bound.dns_mediation_source()); + let remote_host_gateway_ip = remote_boundary + .as_ref() + .and_then(|(bound, _, _)| bound.host_gateway_ip()); + let mut networking = if network_enabled { #[cfg(target_os = "linux")] let proxy_bind_ip = netns @@ -553,9 +637,11 @@ pub async fn run_sandbox( agent_proposals.clone(), workspace_rx.clone(), &upstream_proxy_args, - None, + remote_host_gateway_ip, #[cfg(target_os = "linux")] transparent_runtime, + remote_network_source, + remote_dns_source, ) .await?, ) @@ -563,6 +649,25 @@ pub async fn run_sandbox( None }; + let remote_ready = if let Some((bound, backend_name, ca_file_paths)) = remote_boundary { + ca_file_paths + .lock() + .map_err(|_| miette::miette!("boundary CA path lock is poisoned"))? + .clone_from( + &networking + .as_ref() + .and_then(|runtime| runtime.ca_file_paths.clone()), + ); + let ready = bound + .confirm() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %backend_name, "Isolation boundary enforcement confirmed"); + Some((ready, backend_name)) + } else { + None + }; + #[cfg(target_os = "linux")] let sidecar_control_server = if network_enabled && sidecar_network_enforcement { if !matches!(policy.network.mode, NetworkMode::Proxy) { @@ -850,7 +955,65 @@ pub async fn run_sandbox( }; tokio::pin!(proxy_exited); - let exit_code = if process_enabled { + let exit_code = if let Some((ready, backend_name)) = remote_ready { + let running = ready + .start_agent() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %backend_name, "Isolation boundary agent started"); + let boundary_access = openshell_supervisor_process::delegated::start_boundary_access( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + ssh_socket_path.as_deref(), + false, + networking + .as_ref() + .and_then(|runtime| runtime.ca_file_paths.clone()), + process_enforcement_mode, + running.exec(), + running.port_forward(), + ) + .await?; + info!(backend = %backend_name, "Control-mode access plane started"); + let instance_id = boundary_access.instance_id().to_string(); + let agent = running.agent(); + let wait_agent = agent.clone(); + let wait = async move { + wait_agent + .wait() + .await + .map(|status| match status { + openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code) => { + code + } + openshell_isolation_interface::contract::BoundaryExitStatus::Signaled( + signal, + ) => 128_i32.saturating_add(signal), + }) + .map_err(|error| miette::miette!(error.to_string())) + }; + let exit_code = tokio::select! { + result = wait => result?, + () = &mut proxy_exited => { + let _ = agent.terminate().await; + return Err(miette::miette!( + "control-mode proxy accept loop exited unexpectedly" + )); + } + }; + if let (Some(endpoint), Some(id)) = (openshell_endpoint.as_deref(), sandbox_id.as_deref()) { + openshell_supervisor_process::delegated::report_main_process_completion( + endpoint, + id, + &instance_id, + exit_code, + ) + .await; + } + drop(running); + drop(boundary_access); + exit_code + } else if process_enabled { let ca_file_paths = networking .as_ref() .and_then(|n| n.ca_file_paths.clone()) @@ -1450,7 +1613,11 @@ fn spawn_sidecar_entrypoint_handler( endpoint.clone(), id.clone(), trusted_ssh_socket_path.clone(), - None, + Arc::new( + openshell_supervisor_process::boundary_io::NetnsPortForward::new( + None, None, + ), + ), Some(supervisor_pid), Arc::clone(&terminating), started.instance_id.clone(), diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 7108378654..2f29068712 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -55,17 +55,22 @@ const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; #[cfg(target_os = "linux")] const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; -/// Which supervisor leaves are enabled in this process. +/// Which supervisor role or legacy leaves are enabled in this process. /// -/// Parsed from a comma-separated `--mode` value, e.g. `network`, -/// `process`, or `network,process`. `network-init` is a one-shot setup mode -/// used by the Kubernetes sidecar topology and cannot be combined with other -/// mode components. At least one must be set. +/// `control` is the RFC 0012 logical supervisor outside the workload; +/// `boundary` is its authenticated boundary-local counterpart. Legacy leaf +/// combinations remain accepted while existing deployments migrate. #[derive(Clone, Copy, Debug)] +#[allow( + clippy::struct_excessive_bools, + reason = "legacy leaf combinations coexist with the two migration roles" +)] struct Mode { network: bool, process: bool, network_init: bool, + control: bool, + boundary: bool, } impl std::str::FromStr for Mode { @@ -76,25 +81,36 @@ impl std::str::FromStr for Mode { network: false, process: false, network_init: false, + control: false, + boundary: false, }; for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { match part { "network" => mode.network = true, "process" => mode.process = true, "network-init" => mode.network_init = true, + "control" => { + mode.control = true; + mode.network = true; + mode.process = true; + } + "boundary" => mode.boundary = true, other => { return Err(format!( - "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" + "unknown mode component '{other}' (expected 'control', 'boundary', 'network', 'process', or 'network-init')" )); } } } - if mode.network_init && (mode.network || mode.process) { - return Err("--mode=network-init cannot be combined with other components".into()); + if (mode.control || mode.boundary || mode.network_init) + && s.split(',').filter(|part| !part.trim().is_empty()).count() != 1 + { + return Err("control, boundary, and network-init modes cannot be combined".into()); } - if !mode.network && !mode.process && !mode.network_init { + if !mode.network && !mode.process && !mode.network_init && !mode.boundary { return Err( - "--mode must enable at least one of: network, process, network-init".into(), + "--mode must enable one of: control, boundary, network, process, network-init" + .into(), ); } Ok(mode) @@ -177,7 +193,10 @@ struct Args { #[arg(long, default_value = "8080")] health_port: u16, - /// Which supervisor components to run. Comma-separated list of + /// Supervisor role to run. Drivers use `control` outside the workload and + /// `boundary` inside it. Legacy comma-separated leaf modes remain accepted. + /// + /// Legacy values are a comma-separated list of /// "network" and/or "process". Defaults to both (single-binary /// topology). Use --mode=network for a network-only sidecar, or /// --mode=process for a process-only supervisor when network @@ -186,6 +205,10 @@ struct Args { #[arg(long, default_value = DEFAULT_MODE)] mode: Mode, + /// Protected boundary configuration used only with `--mode=boundary`. + #[arg(long)] + boundary_config: Option, + /// UID that the long-running Kubernetes network sidecar will run as. /// `--mode=network-init` installs nftables rules that exempt this UID. #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] @@ -235,6 +258,22 @@ struct Args { /// re-signed upstream certificates and the sandbox trust bundle. #[arg(long)] upstream_proxy_ca_bundle: Option, + + /// Backend named by the compute driver's topology descriptor. + #[arg(long)] + topology_backend_name: Option, + + /// Isolation Backend interface version named by the topology descriptor. + #[arg(long)] + topology_version: Option, + + /// Base64-encoded opaque topology-descriptor payload. + #[arg(long, conflicts_with = "topology_payload_file")] + topology_payload_base64: Option, + + /// Driver-owned protected file holding the raw topology payload. + #[arg(long)] + topology_payload_file: Option, } /// Internal one-shot command used by the privileged supervisor to validate an @@ -542,6 +581,28 @@ fn main() -> Result<()> { let args = Args::parse(); + if args.mode.boundary { + let console_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); + let _ = tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .try_init(); + let config = args + .boundary_config + .as_deref() + .ok_or_else(|| miette::miette!("--mode=boundary requires --boundary-config"))?; + return openshell_sandbox::run_boundary(config); + } + if args.boundary_config.is_some() { + return Err(miette::miette!( + "--boundary-config is only valid with --mode=boundary" + )); + } + if args.mode.network_init { let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); return run_network_init( @@ -692,6 +753,53 @@ fn main() -> Result<()> { proxy_ca_bundle: args.upstream_proxy_ca_bundle, }; + let topology_payload = match (args.topology_payload_base64, args.topology_payload_file) { + (None, None) => None, + (Some(encoded), None) => { + use base64::Engine as _; + Some( + base64::engine::general_purpose::STANDARD + .decode(encoded) + .into_diagnostic()?, + ) + } + (None, Some(path)) => Some(std::fs::read(&path).map_err(|error| { + miette::miette!("read topology payload {}: {error}", path.display()) + })?), + (Some(_), Some(_)) => unreachable!("clap rejects conflicting payload flags"), + }; + let topology_descriptor = match ( + args.topology_backend_name, + args.topology_version, + topology_payload, + ) { + (None, None, None) => None, + (Some(backend_name), Some(version), Some(payload)) => Some( + openshell_isolation_interface::contract::TopologyDescriptor { + backend_name, + version, + payload, + }, + ), + _ => { + return Err(miette::miette!( + "topology descriptor requires backend name, version, and payload" + )); + } + }; + let admitted_isolation_backend = + std::env::var(openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND).ok(); + if args.mode.control && topology_descriptor.is_none() { + return Err(miette::miette!( + "--mode=control requires a protected topology descriptor" + )); + } + if topology_descriptor.is_some() && !args.mode.control { + return Err(miette::miette!( + "a topology descriptor requires --mode=control" + )); + } + run_sandbox( command, workdir, @@ -711,6 +819,8 @@ fn main() -> Result<()> { args.mode.network, args.mode.process, upstream_proxy_args, + topology_descriptor, + admitted_isolation_backend, ) .await })?; @@ -815,6 +925,30 @@ mod tests { assert!(err.contains("cannot be combined")); } + #[test] + fn mode_parses_control_as_combined_supervisor() { + let mode = "control".parse::().unwrap(); + assert!(mode.control); + assert!(mode.network); + assert!(mode.process); + assert!(!mode.boundary); + } + + #[test] + fn mode_parses_boundary_as_standalone_leaf() { + let mode = "boundary".parse::().unwrap(); + assert!(mode.boundary); + assert!(!mode.control); + assert!(!mode.network); + assert!(!mode.process); + } + + #[test] + fn mode_rejects_control_or_boundary_combinations() { + assert!("control,network".parse::().is_err()); + assert!("boundary,process".parse::().is_err()); + } + #[test] fn mode_rejects_empty_value() { let err = "".parse::().unwrap_err(); diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index a828f75fba..f5537d28f3 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -19,6 +19,7 @@ pub mod procfs; pub mod proxy; pub mod run; pub mod sigv4; +mod spiffe_endpoint; mod token_grant; pub mod upstream_proxy; diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 63aa2c2c70..b9d9316d2e 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -810,6 +810,7 @@ impl OpaEngine { /// generation comparison and callback linearizes state derived from an OPA /// snapshot with every policy reload and fail-closed transition. Callers /// must not perform I/O or other long-running work in `operation`. + #[allow(dead_code)] pub(crate) fn with_current_generation( &self, expected_generation: u64, diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs index ad6095efa9..15ef423d83 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -9,6 +9,7 @@ use super::{PolicyDnsService, SocketTrustedResolver, wire}; use crate::opa::OpaEngine; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::net::set_tcp_nodelay_best_effort; +use openshell_isolation_interface::contract::{DnsMediationSource, DnsTransport}; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; @@ -63,6 +64,75 @@ pub(crate) struct PolicyDnsRuntime { } impl PolicyDnsRuntime { + /// Start policy DNS over an isolation-backend exchange source. No UDP or + /// TCP listener is bound in the supervisor namespace. + pub(crate) fn start_mediated( + policy: Arc, + source: Arc, + trusted_host_gateway: Option, + config: PolicyDnsRuntimeConfig, + mut engine_ready: tokio::sync::watch::Receiver, + ) -> Result { + let upstream = trusted_resolver_from_resolv_conf()?; + let store = Arc::new(ResolvedEndpointStore::new( + StoreConfig::new(config.pools, MAX_MAPPINGS) + .map_err(|error| miette::miette!(error.to_string()))?, + )); + let service = Arc::new(PolicyDnsService::new( + policy, + SocketTrustedResolver::new(upstream), + store.clone(), + trusted_host_gateway, + )); + let task = tokio::spawn(async move { + if engine_ready.wait_for(|ready| *ready).await.is_err() { + return; + } + loop { + let Ok(query) = source.accept().await else { + return; + }; + let service = service.clone(); + tokio::spawn(async move { + let response = match query.transport { + DnsTransport::Udp => { + wire::handle_udp_query_with_ipv6(&service, &query.request, false).await + } + DnsTransport::Tcp => { + wire::handle_tcp_query_with_ipv6(&service, &query.request, false).await + } + } + .map_err(|error| { + openshell_isolation_interface::contract::BackendError::Process(format!( + "policy DNS response failed: {error}" + )) + }); + let _ = query.response.send(response); + }); + } + }); + let expiry_store = store.clone(); + let expiry_task = tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + let _ = expiry_store.expire(std::time::Instant::now()); + } + }); + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "ready") + .message("Policy DNS connected to isolation boundary") + .build() + ); + Ok(Self { + store, + tasks: vec![task, expiry_task], + }) + } + pub(crate) fn start( policy: Arc, udp: tokio::net::UdpSocket, diff --git a/crates/openshell-supervisor-network/src/policy_dns/store.rs b/crates/openshell-supervisor-network/src/policy_dns/store.rs index ea2cd7c0f8..439cdcfa36 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/store.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/store.rs @@ -73,6 +73,17 @@ pub(crate) struct MappingLookup { } impl MappingLookup { + pub(crate) fn pinned_addresses(&self) -> Vec { + let mut seen = HashSet::new(); + self.record + .contracts + .iter() + .filter(|contract| contract.port == self.port) + .flat_map(|contract| contract.pinned_addresses.iter().copied()) + .filter(|address| seen.insert(*address)) + .collect() + } + pub(crate) fn endpoint_ids(&self) -> impl Iterator { self.record .contracts diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 51dd0007af..5848cdbfd9 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -10,8 +10,9 @@ mod relay; use crate::identity::BinaryIdentityCache; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; +use crate::policy_dns::ResolvedEndpointStore; #[cfg(target_os = "linux")] -use crate::policy_dns::{MappingLookupError, PolicyEndpointId, ResolvedEndpointStore}; +use crate::policy_dns::{MappingLookupError, PolicyEndpointId}; use crate::policy_local::{POLICY_LOCAL_HOST, PolicyLocalContext}; use crate::upstream_proxy::{self, UpstreamProxyConfig}; use miette::{IntoDiagnostic, Result}; @@ -24,6 +25,10 @@ use openshell_core::net::{ use openshell_core::policy::ProxyPolicy; use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; use openshell_core::secrets::{self, SecretResolver, rewrite_header_line_checked}; +use openshell_isolation_interface::contract::{ + BinaryIdentity as ContractBinaryIdentity, BoundaryDuplexStream, NetworkMediationSource, + ResolveError, +}; use openshell_ocsf::{ ActionId, ActivityId, AiModel, ApiActivityBuilder, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, Process, SeverityId, StatusId, @@ -36,16 +41,24 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; use tokio::io::{ - AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, AsyncWriteExt, + AsyncBufReadExt, AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, + AsyncWriteExt, }; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{debug, warn}; +type ProxyClient = tokio::io::BufReader; + +enum ProxyAcceptError { + Listener(std::io::Error), + Source(openshell_isolation_interface::contract::BackendError), +} + use self::destination::{ - DestinationDenial, DestinationDenialKind, DestinationRequest, build_validation_plan, - validate_destination, + DestinationDenial, DestinationDenialKind, DestinationRequest, build_pinned_validation_plan, + build_validation_plan, validate_destination, }; use self::egress::{ EgressDecision, EgressIntent, EndpointDecision, IdentityUnavailableReason, L7ConfigSnapshot, @@ -256,6 +269,8 @@ impl ProxyHandle { engine_ready: tokio::sync::watch::Receiver, upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, backend_host_gateway: Option, + network_mediation_source: Option>, + policy_dns_store: Option>, ) -> Result { // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no @@ -271,15 +286,27 @@ impl ProxyHandle { )); } - let listener = TcpListener::bind(http_addr).await.into_diagnostic()?; - let local_addr = listener.local_addr().into_diagnostic()?; + let source_backed = network_mediation_source.is_some(); + let listener = if source_backed { + None + } else { + Some(TcpListener::bind(http_addr).await.into_diagnostic()?) + }; + let local_addr = match listener.as_ref() { + Some(listener) => listener.local_addr().into_diagnostic()?, + None => http_addr, + }; { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Listen) .severity(SeverityId::Informational) .status(StatusId::Success) .dst_endpoint(Endpoint::from_ip(local_addr.ip(), local_addr.port())) - .message(format!("Proxy listening on {local_addr}")) + .message(if source_backed { + "Proxy consuming isolation-boundary streams".to_string() + } else { + format!("Proxy listening on {local_addr}") + }) .build(); ocsf_emit!(event); } @@ -373,11 +400,39 @@ impl ProxyHandle { let mut consecutive_resource_errors: u32 = 0; let mut consecutive_unknown_errors: u32 = 0; loop { - match listener.accept().await { - Ok((stream, _addr)) => { + let accepted = if let Some(source) = network_mediation_source.as_ref() { + source + .accept() + .await + .map(|connection| { + ( + connection.stream, + Some(connection.binary_identity), + None, + connection.destination, + ) + }) + .map_err(ProxyAcceptError::Source) + } else { + let listener = listener + .as_ref() + .expect("listener exists without a mediation source"); + listener + .accept() + .await + .map(|(stream, _)| { + set_tcp_nodelay_best_effort(&stream); + let workload_addr = stream.peer_addr().ok(); + let proxy_addr = stream.local_addr().ok(); + let stream: BoundaryDuplexStream = Box::new(stream); + (stream, None, workload_addr.zip(proxy_addr), None) + }) + .map_err(ProxyAcceptError::Listener) + }; + match accepted { + Ok((stream, supplied_identity, socket_addrs, transparent_destination)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; - set_tcp_nodelay_best_effort(&stream); let opa = opa_engine.clone(); let cache = identity_cache.clone(); let spid = entrypoint_pid.clone(); @@ -389,6 +444,7 @@ impl ProxyHandle { let backend_gw = backend_host_gateway.clone(); let up_proxy = upstream_proxy.clone(); let credentials = provider_credentials.clone(); + let dns_store = policy_dns_store.clone(); let resolver = provider_credentials .as_ref() .and_then(ProviderCredentialState::resolver); @@ -401,8 +457,12 @@ impl ProxyHandle { let atx = activity_tx.clone(); tokio::spawn(async move { #[allow(clippy::large_futures)] - if let Err(err) = handle_tcp_connection( - stream, + if let Err(err) = handle_mediated_connection( + tokio::io::BufReader::new(stream), + supplied_identity, + socket_addrs, + transparent_destination, + dns_store, opa, cache, spid, @@ -431,7 +491,19 @@ impl ProxyHandle { } }); } - Err(err) => { + Err(ProxyAcceptError::Source(err)) => { + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message(format!( + "Network-mediation source failed; proxy accept loop exiting: {err}" + )) + .build(); + ocsf_emit!(event); + break; + } + Err(ProxyAcceptError::Listener(err)) => { match classify_accept_error( &err, &mut consecutive_resource_errors, @@ -469,7 +541,7 @@ impl ProxyHandle { }); Ok(Self { - http_addr: Some(local_addr), + http_addr: (!source_backed).then_some(local_addr), join, exited_rx: Some(exited_rx), }) @@ -1196,21 +1268,24 @@ fn middleware_uninspectable_gate( Ok(crate::l7::middleware::uninspectable_traffic_gate(&chain)) } -async fn peek_tunnel_protocol(client: &TcpStream) -> Result> { - let mut peek_buf = [0u8; TUNNEL_PROTOCOL_PEEK_BYTES]; +async fn peek_tunnel_protocol(client: &mut C) -> Result> +where + C: tokio::io::AsyncBufRead + Unpin, +{ let deadline = tokio::time::Instant::now() + TUNNEL_PROTOCOL_PEEK_TIMEOUT; loop { - let n = client.peek(&mut peek_buf).await.into_diagnostic()?; - if n == 0 { + let available = client.fill_buf().await.into_diagnostic()?; + if available.is_empty() { return Ok(None); } - let peek = &peek_buf[..n]; + let n = available.len().min(TUNNEL_PROTOCOL_PEEK_BYTES); + let peek = &available[..n]; let protocol = classify_tunnel_protocol(peek); if protocol != TunnelProtocol::Unsupported || !could_be_supported_tunnel_protocol_prefix(peek) - || n == peek_buf.len() + || n == TUNNEL_PROTOCOL_PEEK_BYTES || tokio::time::Instant::now() >= deadline { return Ok(Some(protocol)); @@ -1588,8 +1663,8 @@ fn build_forward_destination_deny_ocsf_event( } #[allow(clippy::too_many_arguments)] -async fn deny_connect_destination( - client: &mut TcpStream, +async fn deny_connect_destination( + client: &mut C, denial: &DestinationDenial, peer_addr: SocketAddr, host: &str, @@ -1601,7 +1676,10 @@ async fn deny_connect_destination( decision: &EgressDecision, denial_tx: &Option>, activity_tx: &Option, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ let detail = destination_denial_detail(denial.kind); ocsf_emit!(build_connect_destination_deny_ocsf_event( denial, peer_addr, host, port, binary, pid, ancestors, cmdline, @@ -1634,8 +1712,8 @@ async fn deny_connect_destination( } #[allow(clippy::too_many_arguments)] -async fn deny_forward_destination( - client: &mut TcpStream, +async fn deny_forward_destination( + client: &mut C, denial: &DestinationDenial, peer_addr: SocketAddr, method: &str, @@ -1650,7 +1728,10 @@ async fn deny_forward_destination( decision: &EgressDecision, denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ let detail = destination_denial_detail(denial.kind); ocsf_emit!(build_forward_destination_deny_ocsf_event( denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, @@ -1685,9 +1766,10 @@ async fn deny_forward_destination( // Many distinct, non-related context parameters are required for a CONNECT // dispatch; bundling them into a struct would just shift the noise into call // sites. +#[cfg(test)] #[allow(clippy::too_many_arguments)] async fn handle_tcp_connection( - mut client: TcpStream, + client: TcpStream, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -1710,6 +1792,123 @@ async fn handle_tcp_connection( denial_tx: Option>, activity_tx: Option, ) -> Result<()> { + let socket_addrs = client.peer_addr().ok().zip(client.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(client); + Box::pin(handle_mediated_connection( + tokio::io::BufReader::new(stream), + None, + socket_addrs, + None, + None, + opa_engine, + identity_cache, + entrypoint_pid, + tls_state, + inference_ctx, + policy_local_ctx, + agent_proposals, + backend_host_gateway, + trusted_host_gateway, + upstream_proxy, + provider_credentials, + secret_resolver, + dynamic_credentials, + denial_tx, + activity_tx, + )) + .await +} + +/// Adapt a transparent application stream to the existing CONNECT pipeline. +/// The synthetic CONNECT request is supervisor-owned and its successful 200 +/// response is consumed before bytes are returned to the workload. +fn virtual_connect_stream( + workload: BoundaryDuplexStream, + authority: String, +) -> BoundaryDuplexStream { + let (handler, bridge) = tokio::io::duplex(64 * 1024); + let (mut bridge_read, mut bridge_write) = tokio::io::split(bridge); + let (mut workload_read, mut workload_write) = tokio::io::split(workload); + tokio::spawn(async move { + let request = format!("CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n"); + if bridge_write.write_all(request.as_bytes()).await.is_ok() { + let _ = tokio::io::copy(&mut workload_read, &mut bridge_write).await; + } + let _ = bridge_write.shutdown().await; + }); + tokio::spawn(async move { + let mut header = Vec::with_capacity(256); + let mut byte = [0_u8; 1]; + while header.len() < MAX_HEADER_BYTES { + match bridge_read.read(&mut byte).await { + Ok(0) | Err(_) => return, + Ok(_) => header.push(byte[0]), + } + if header.ends_with(b"\r\n\r\n") { + break; + } + } + if !header.starts_with(b"HTTP/1.1 200 ") && !header.starts_with(b"HTTP/1.0 200 ") { + let _ = workload_write.shutdown().await; + return; + } + let _ = tokio::io::copy(&mut bridge_read, &mut workload_write).await; + let _ = workload_write.shutdown().await; + }); + Box::new(handler) +} + +#[allow(clippy::too_many_arguments)] +async fn handle_mediated_connection( + mut client: ProxyClient, + supplied_identity: Option>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, + transparent_destination: Option, + policy_dns_store: Option>, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + tls_state: Option>, + inference_ctx: Option>, + policy_local_ctx: Option>, + agent_proposals: openshell_core::proposals::AgentProposals, + backend_host_gateway: Arc>, + trusted_host_gateway: Arc>, + upstream_proxy: Arc>, + provider_credentials: Option, + secret_resolver: Option>, + dynamic_credentials: Option< + Arc< + std::sync::RwLock< + std::collections::HashMap, + >, + >, + >, + denial_tx: Option>, + activity_tx: Option, +) -> Result<()> { + let transparent_mapping = if let Some(destination) = transparent_destination { + let store = policy_dns_store + .as_ref() + .ok_or_else(|| miette::miette!("transparent connection arrived without policy DNS"))?; + let mapping = store + .lookup( + destination.ip(), + destination.port(), + opa_engine.current_generation(), + std::time::Instant::now(), + ) + .map_err(|error| miette::miette!("transparent destination denied: {error}"))?; + let authority = format!( + "{}:{}", + mapping.record.normalized_name.as_str(), + destination.port() + ); + client = tokio::io::BufReader::new(virtual_connect_stream(client.into_inner(), authority)); + Some(mapping) + } else { + None + }; let mut buf = vec![0u8; MAX_HEADER_BYTES]; let mut used = 0usize; @@ -1762,6 +1961,8 @@ async fn handle_tcp_connection( &buf[..], used, &mut client, + supplied_identity.as_ref(), + socket_addrs, opa_engine, identity_cache, entrypoint_pid, @@ -1809,22 +2010,31 @@ async fn handle_tcp_connection( return Ok(()); } - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); // Evaluate OPA policy with process-identity binding. // Wrapped in spawn_blocking because identity resolution does heavy sync I/O: // /proc scanning + SHA256 hashing of binaries (e.g. node at 124MB). - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); let intent = EgressIntent::connect(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity.as_ref() { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -1962,6 +2172,13 @@ async fn handle_tcp_connection( return Ok(()); } } + if let Some(mapping) = transparent_mapping.as_ref() { + decision.endpoint.destination = Some( + build_pinned_validation_plan(mapping.pinned_addresses()).map_err(|denial| { + miette::miette!("transparent destination mapping denied: {}", denial.reason) + })?, + ); + } let destination_plan = decision .endpoint .destination @@ -2211,7 +2428,7 @@ async fn handle_tcp_connection( // Auto-detect the tunnel payload. L7-configured endpoints must only // enter relays that can enforce their configured protocol; unsupported // bytes fail closed below instead of falling through to raw relay. - let Some(tunnel_protocol) = peek_tunnel_protocol(&client).await? else { + let Some(tunnel_protocol) = peek_tunnel_protocol(&mut client).await? else { return Ok(()); }; @@ -2786,6 +3003,77 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres } } +/// Evaluate an egress intent using identity already bound to the accepted +/// connection by an isolation backend. This is the RFC 0012 path; legacy +/// listeners continue to resolve through procfs in `authorize_egress_intent`. +fn authorize_supplied_identity( + engine: &OpaEngine, + intent: EgressIntent, + identity: &Result, +) -> EgressDecision { + let deny = |reason: String, + binary: Option, + ancestors: Vec, + cmdline_paths: Vec| EgressDecision { + intent: intent.clone(), + action: NetworkAction::Deny { reason }, + policy_generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), + endpoint: EndpointDecision::default(), + binary, + binary_pid: None, + ancestors, + cmdline_paths, + }; + + let identity = match identity { + Ok(identity) => identity, + Err(error) => { + return deny( + format!("backend identity resolution failed: {error}"), + None, + vec![], + vec![], + ); + } + }; + let Some(digest) = identity.binary_digest else { + return deny( + "backend identity did not include the required binary digest".to_string(), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ); + }; + let input = crate::opa::NetworkInput { + host: intent.destination.host.clone(), + port: intent.destination.port, + binary_path: identity.binary_path.clone(), + binary_sha256: digest.to_string(), + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }; + match engine.evaluate_network_action_with_generation(&input) { + Ok((action, generation)) => EgressDecision { + intent, + action, + policy_generation: generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), + binary: Some(identity.binary_path.clone()), + binary_pid: None, + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }, + Err(error) => deny( + format!("policy evaluation error: {error}"), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ), + } +} + /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] fn authorize_egress_intent( @@ -2830,13 +3118,16 @@ const INITIAL_INFERENCE_BUF: usize = 65536; /// /// Returns [`InferenceOutcome::Routed`] if at least one request was successfully /// routed, or [`InferenceOutcome::Denied`] with a reason for all denial cases. -async fn handle_inference_interception( - client: TcpStream, +async fn handle_inference_interception( + client: S, host: &str, port: u16, tls_state: Option<&Arc>, inference_ctx: Option<&Arc>, -) -> Result { +) -> Result +where + S: TokioAsyncRead + TokioAsyncWrite + Unpin + Send, +{ let Some(ctx) = inference_ctx else { return Ok(InferenceOutcome::Denied { reason: "cluster inference context not configured".to_string(), @@ -3425,13 +3716,16 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette ocsf_emit!(event); } -async fn reject_stale_connect_policy( - client: &mut TcpStream, +async fn reject_stale_connect_policy( + client: &mut C, host: &str, port: u16, activity_tx: Option<&ActivitySender>, error: miette::Report, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ warn!( host, port, @@ -4826,7 +5120,9 @@ async fn handle_forward_proxy( target_uri: &str, buf: &[u8], used: usize, - client: &mut TcpStream, + client: &mut ProxyClient, + supplied_identity: Option<&Result>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -4931,19 +5227,27 @@ async fn handle_forward_proxy( canonicalize_forward_host_header(&buf[..used], &canonical_authority)?; // 2. Evaluate OPA policy (same identity binding as CONNECT) - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); - - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); let intent = EgressIntent::forward_http(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -5893,6 +6197,17 @@ async fn handle_forward_proxy( ), ) .await?; + client.shutdown().await.into_diagnostic()?; + let mut discard = [0_u8; 1024]; + let _ = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + match client.read(&mut discard).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + }) + .await; } else { respond( client, @@ -6169,8 +6484,9 @@ fn normalize_host(raw_host: &str) -> &str { raw_host.strip_suffix('.').unwrap_or(raw_host) } -async fn respond(client: &mut TcpStream, bytes: &[u8]) -> Result<()> { +async fn respond(client: &mut (impl TokioAsyncWrite + Unpin), bytes: &[u8]) -> Result<()> { client.write_all(bytes).await.into_diagnostic()?; + client.flush().await.into_diagnostic()?; Ok(()) } @@ -6309,11 +6625,14 @@ const TLS_TERMINATION_UNAVAILABLE_DETAIL: &str = "TLS termination unavailable (C /// HTTP status (the flaw this replaces). Returns `true` when the connection was /// refused (the caller must stop) and `false` when the caller should proceed to /// establish the tunnel. -async fn refuse_connect_when_tls_unavailable( - client: &mut TcpStream, +async fn refuse_connect_when_tls_unavailable( + client: &mut C, tls_state_present: bool, effective_tls_skip: bool, -) -> Result { +) -> Result +where + C: TokioAsyncWrite + Unpin, +{ if tls_state_present || effective_tls_skip { return Ok(false); } @@ -6364,6 +6683,44 @@ mod tests { use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; + struct FailedMediationSource; + + #[tokio::test] + async fn virtual_connect_is_portless_and_hides_the_synthetic_handshake() { + let (workload, mut workload_peer) = tokio::io::duplex(1024); + let mut handler = virtual_connect_stream(Box::new(workload), "api.example.com:443".into()); + + workload_peer.write_all(b"client-tls").await.unwrap(); + let mut request = vec![0_u8; 128]; + let length = handler.read(&mut request).await.unwrap(); + let request = &request[..length]; + assert!(request.starts_with(b"CONNECT api.example.com:443 HTTP/1.1\r\n")); + + handler + .write_all(b"HTTP/1.1 200 Connection Established\r\n\r\nserver-tls") + .await + .unwrap(); + let mut response = [0_u8; 10]; + workload_peer.read_exact(&mut response).await.unwrap(); + assert_eq!(&response, b"server-tls"); + } + + #[async_trait::async_trait] + impl NetworkMediationSource for FailedMediationSource { + async fn accept( + &self, + ) -> std::result::Result< + openshell_isolation_interface::contract::MediatedConnection, + openshell_isolation_interface::contract::BackendError, + > { + Err( + openshell_isolation_interface::contract::BackendError::Unavailable( + "test source unavailable".to_string(), + ), + ) + } + } + struct DenyWebSocketPreflight; #[tonic::async_trait] @@ -6551,6 +6908,49 @@ network_policies: {} client.await.unwrap() } + #[tokio::test] + async fn terminal_mediation_source_failure_stops_proxy() { + let policy = include_str!("../data/sandbox-policy.rego"); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + policy, + "network_policies: {}", + true, + ) + .expect("engine"), + ); + let (_ready_tx, ready_rx) = tokio::sync::watch::channel(true); + let mut handle = ProxyHandle::start_with_bind_addr( + &ProxyPolicy { http_addr: None }, + Some(([127, 0, 0, 1], 3128).into()), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(1)), + None, + None, + None, + None, + None, + None, + ready_rx, + &upstream_proxy::UpstreamProxyArgs::default(), + None, + Some(Arc::new(FailedMediationSource)), + None, + ) + .await + .expect("proxy starts before source accept"); + let exited = handle + .take_exit_receiver() + .expect("proxy exposes its exit receiver"); + + tokio::time::timeout(std::time::Duration::from_secs(1), exited) + .await + .expect("source failure must stop the proxy") + .expect_err("proxy task drops the exit sender"); + assert!(handle.join.is_finished()); + } + #[tokio::test] async fn malformed_forward_headers_are_rejected_before_route_or_middleware_dispatch() { for host in ["api.example.com", "unmatched.example.com"] { @@ -6636,7 +7036,13 @@ network_policies: .expect("read proxy response"); response }); - let (mut proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let (proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let socket_addrs = proxy_connection + .peer_addr() + .ok() + .zip(proxy_connection.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(proxy_connection); + let mut proxy_connection = tokio::io::BufReader::new(stream); tokio::time::timeout( std::time::Duration::from_secs(30), @@ -6646,6 +7052,8 @@ network_policies: request.as_bytes(), request.len(), &mut proxy_connection, + None, + socket_addrs, engine, Arc::new(BinaryIdentityCache::new()), Arc::new(AtomicU32::new(std::process::id())), @@ -6771,7 +7179,13 @@ network_policies: .await .unwrap(); }); - let (mut proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let (proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let socket_addrs = proxy_connection + .peer_addr() + .ok() + .zip(proxy_connection.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(proxy_connection); + let mut proxy_connection = tokio::io::BufReader::new(stream); let handler = tokio::spawn(async move { handle_forward_proxy( @@ -6780,6 +7194,8 @@ network_policies: request.as_bytes(), request.len(), &mut proxy_connection, + None, + socket_addrs, engine, Arc::new(BinaryIdentityCache::new()), Arc::new(AtomicU32::new(std::process::id())), @@ -7365,14 +7781,14 @@ network_policies: let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let mut client = TcpStream::connect(addr).await.unwrap(); - let (server, _) = listener.accept().await.unwrap(); + let (mut server, _) = listener.accept().await.unwrap(); client .write_all(crate::l7::rest::HTTP2_PRIOR_KNOWLEDGE_PREFACE) .await .unwrap(); - let protocol = peek_tunnel_protocol(&server) + let protocol = peek_tunnel_protocol(&mut tokio::io::BufReader::new(&mut server)) .await .expect("peek should succeed") .expect("client sent bytes"); diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs index 314596b048..55c4dd9099 100644 --- a/crates/openshell-supervisor-network/src/proxy/egress.rs +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![allow(dead_code)] + //! Transport-neutral egress inputs and authorization results. //! //! Explicit proxy adapters normalize their protocol-specific request into an diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 0f29e331ff..e761bb57e7 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -37,6 +37,7 @@ use crate::l7::tls::{ use crate::opa::OpaEngine; use crate::policy_local::PolicyLocalContext; use crate::proxy::ProxyHandle; +use openshell_isolation_interface::contract::{DnsMediationSource, NetworkMediationSource}; #[cfg(target_os = "linux")] pub struct TransparentRuntimeSetup { @@ -155,6 +156,7 @@ pub struct Networking { /// loop so it can publish updated `SandboxPolicy` snapshots that the /// `policy.local` route handler returns to the workload. pub policy_local_ctx: Arc, + _mediated_policy_dns: Option, #[cfg(target_os = "linux")] _policy_dns: Option, #[cfg(target_os = "linux")] @@ -198,6 +200,8 @@ pub async fn run_networking( upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, host_gateway_ip: Option, #[cfg(target_os = "linux")] transparent_runtime: Option, + network_mediation_source: Option>, + dns_mediation_source: Option>, ) -> Result { // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so @@ -403,6 +407,21 @@ pub async fn run_networking( (None, None) }; + let mediated_policy_dns = if let Some(source) = dns_mediation_source { + let engine = opa_engine + .cloned() + .ok_or_else(|| miette::miette!("Mediated DNS requires an OPA engine"))?; + Some(crate::policy_dns::PolicyDnsRuntime::start_mediated( + engine, + source, + None, + crate::policy_dns::PolicyDnsRuntimeConfig::for_epoch(0)?, + engine_ready_rx.clone(), + )?) + } else { + None + }; + let proxy_handle = if matches!(policy.network.mode, NetworkMode::Proxy) { let proxy_policy = policy.network.proxy.as_ref().ok_or_else(|| { miette::miette!("Network mode is set to proxy but no proxy configuration was provided") @@ -450,6 +469,10 @@ pub async fn run_networking( engine_ready_rx, upstream_proxy_args, host_gateway_ip, + network_mediation_source, + mediated_policy_dns + .as_ref() + .map(|runtime| runtime.store.clone()), ) .await?; Some(proxy_handle) @@ -495,6 +518,7 @@ pub async fn run_networking( proxy: proxy_handle, ca_file_paths, policy_local_ctx, + _mediated_policy_dns: mediated_policy_dns, #[cfg(target_os = "linux")] _policy_dns: policy_dns, #[cfg(target_os = "linux")] diff --git a/crates/openshell-supervisor-network/src/spiffe_endpoint.rs b/crates/openshell-supervisor-network/src/spiffe_endpoint.rs new file mode 100644 index 0000000000..b3b6816f1e --- /dev/null +++ b/crates/openshell-supervisor-network/src/spiffe_endpoint.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::Path; + +/// Convert a path to a SPIFFE Workload API endpoint URL. +/// +/// If the path already has a scheme (`unix:` or `tcp:`), use it as-is. +/// Otherwise, assume it is a Unix socket path and prepend `unix:`. +#[allow(dead_code)] +pub fn workload_api_endpoint(path: &Path) -> String { + let path = path.to_string_lossy(); + if path.starts_with("unix:") || path.starts_with("tcp:") { + path.into_owned() + } else { + format!("unix:{path}") + } +} diff --git a/crates/openshell-supervisor-process/src/boundary_exec.rs b/crates/openshell-supervisor-process/src/boundary_exec.rs index eea12cac38..8b3f81f92a 100644 --- a/crates/openshell-supervisor-process/src/boundary_exec.rs +++ b/crates/openshell-supervisor-process/src/boundary_exec.rs @@ -130,7 +130,8 @@ impl LocalBoundaryExec { self.enforcement_mode, #[cfg(target_os = "linux")] prepared, - ); + ) + .map_err(|error| BackendError::Process(error.to_string()))?; #[cfg(target_os = "linux")] let mut child_registry = crate::managed_children::lock(); let mut child = command @@ -232,7 +233,8 @@ impl LocalBoundaryExec { self.enforcement_mode, #[cfg(target_os = "linux")] prepared, - ); + ) + .map_err(|error| BackendError::Process(error.to_string()))?; #[cfg(target_os = "linux")] let mut child_registry = crate::managed_children::lock(); let mut child = command diff --git a/crates/openshell-supervisor-process/src/boundary_io.rs b/crates/openshell-supervisor-process/src/boundary_io.rs index fab37a0062..c477cc7b90 100644 --- a/crates/openshell-supervisor-process/src/boundary_io.rs +++ b/crates/openshell-supervisor-process/src/boundary_io.rs @@ -21,7 +21,7 @@ use openshell_isolation_interface::contract::{ BackendError, BoundaryDuplexStream, BoundaryPortForward, LoopbackTarget, }; use std::collections::HashMap; -use std::os::fd::{AsRawFd, OwnedFd}; +use std::os::fd::OwnedFd; use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Mutex}; @@ -204,13 +204,9 @@ impl BoundaryPortForward for NetnsPortForward { runtime.ensure_active()?; } let addr = std::net::SocketAddr::new(target.host(), target.port()); - let addr_string = addr.to_string(); - let stream = crate::ssh::connect_in_netns( - &addr_string, - self.netns_fd.as_deref().map(AsRawFd::as_raw_fd), - ) - .await - .map_err(|e| BackendError::Process(format!("port-forward connect to {addr}: {e}")))?; + let stream = crate::ssh::connect_in_netns(addr, self.netns_fd.clone()) + .await + .map_err(|e| BackendError::Process(format!("port-forward connect to {addr}: {e}")))?; if let Some(runtime) = &self.runtime { runtime.ensure_active()?; } diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs new file mode 100644 index 0000000000..9e5defd20b --- /dev/null +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -0,0 +1,501 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Additive process and access-plane assembly for a delegated boundary. +//! +//! The established co-located [`crate::run::run_process`] path remains intact. +//! This module exposes the same process primitives to `openshell-sandbox +//! --mode=boundary` and lets `--mode=control` host SSH and gateway relays over +//! placement-neutral boundary interfaces. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::time::Duration; + +use miette::{IntoDiagnostic as _, Result}; +use openshell_core::policy::{NetworkMode, SandboxPolicy}; +use openshell_core::proposals::AgentProposals; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward}; +use openshell_ocsf::{ + ActionId, ActivityId, AppLifecycleBuilder, DispositionId, LaunchTypeId, Process as OcsfProcess, + ProcessActivityBuilder, SeverityId, StatusId, ocsf_emit, +}; + +#[cfg(target_os = "linux")] +use crate::netns::NetworkNamespace; +use crate::process::{ + ProcessEnforcementMode, ProcessHandle, ProcessStatus, ResolvedProcessIdentity, + ResolvedWorkspace, +}; + +fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { + openshell_ocsf::ctx::ctx() +} + +/// Host-side SSH and gateway-session tasks for an already-running boundary. +pub struct BoundaryAccess { + instance_id: String, + terminating: Arc, + ssh_task: Option>, + session_task: Option>, +} + +impl BoundaryAccess { + /// Stable supervisor instance ID used for lifecycle reporting. + #[must_use] + pub fn instance_id(&self) -> &str { + &self.instance_id + } +} + +impl Drop for BoundaryAccess { + fn drop(&mut self) { + self.terminating.store(true, Ordering::Release); + if let Some(task) = self.ssh_task.take() { + task.abort(); + } + if let Some(task) = self.session_task.take() { + task.abort(); + } + } +} + +/// Start the control-owned access plane using boundary-supplied exec and +/// loopback-forwarding capabilities. +#[allow(clippy::too_many_arguments)] +pub async fn start_boundary_access( + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + ssh_socket_path: Option<&str>, + shared_ssh_socket: bool, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + enforcement_mode: ProcessEnforcementMode, + boundary_exec: Arc, + port_forward: Arc, +) -> Result { + let instance_id = uuid::Uuid::new_v4().to_string(); + let terminating = Arc::new(AtomicBool::new(false)); + let Some(ssh_socket_path) = ssh_socket_path.map(std::path::PathBuf::from) else { + return Ok(BoundaryAccess { + instance_id, + terminating, + ssh_task: None, + session_task: None, + }); + }; + + let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); + let listen_path = ssh_socket_path.clone(); + let ssh_port_forward = port_forward.clone(); + let ssh_task = tokio::spawn(async move { + if let Err(error) = crate::ssh::run_ssh_server( + listen_path, + ssh_ready_tx, + ca_file_paths, + enforcement_mode, + shared_ssh_socket, + ssh_port_forward, + boundary_exec, + None, + ) + .await + { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Critical) + .status(StatusId::Failure) + .message(format!("SSH server failed: {error}")) + .build() + ); + } + }); + + match tokio::time::timeout(Duration::from_secs(10), ssh_ready_rx).await { + Ok(Ok(Ok(()))) => {} + Ok(Ok(Err(error))) => { + ssh_task.abort(); + return Err(error.context("SSH server failed during startup")); + } + Ok(Err(_)) => { + ssh_task.abort(); + return Err(miette::miette!( + "SSH server task ended before signaling readiness" + )); + } + Err(_) => { + ssh_task.abort(); + return Err(miette::miette!( + "SSH server did not start within 10 seconds" + )); + } + } + + let session_task = match (openshell_endpoint, sandbox_id) { + (Some(endpoint), Some(id)) => Some(crate::supervisor_session::spawn( + endpoint.to_string(), + id.to_string(), + ssh_socket_path, + port_forward, + None, + terminating.clone(), + instance_id.clone(), + )), + _ => None, + }; + + Ok(BoundaryAccess { + instance_id, + terminating, + ssh_task: Some(ssh_task), + session_task, + }) +} + +/// Spawn the admitted workload without placing the gateway or policy authority +/// inside its boundary. +#[allow(clippy::too_many_arguments, clippy::implicit_hasher)] +pub async fn spawn_workload( + program: &str, + args: &[String], + workdir: Option<&str>, + timeout_secs: u64, + interactive: bool, + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + _ssh_socket_path: Option, + _shared_ssh_socket: bool, + policy: &SandboxPolicy, + resolved_process_identity: ResolvedProcessIdentity, + enforcement_mode: ProcessEnforcementMode, + entrypoint_pid: Arc, + entrypoint_started_tx: Option>, + provider_credentials: ProviderCredentialState, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + agent_proposals: AgentProposals, + #[cfg(target_os = "linux")] netns: Option<&NetworkNamespace>, + #[cfg(target_os = "linux")] bypass_denial_tx: Option< + tokio::sync::mpsc::UnboundedSender, + >, + #[cfg(target_os = "linux")] bypass_activity_tx: Option< + openshell_core::activity::ActivitySender, + >, + boundary_runtime: Option>, + instance_id: Option, +) -> Result { + let workspace = ResolvedWorkspace::new(workdir.map(str::to_string), false); + + #[cfg(unix)] + if enforcement_mode.uses_privileged_process_setup() { + crate::process::update_sandbox_passwd_entries()?; + crate::process::validate_sandbox_user_with_identity(policy, resolved_process_identity)?; + crate::process::validate_sandbox_group_with_identity(policy, resolved_process_identity)?; + crate::process::prepare_filesystem_with_identity( + policy, + resolved_process_identity, + workspace.root(), + workspace.home().is_some(), + )?; + } + + crate::run::install_initial_agent_skill(sandbox_id, openshell_endpoint, &agent_proposals).await; + + #[cfg(target_os = "linux")] + crate::process::prepare_supervisor_identity_mount_namespace_from_env()?; + crate::sandbox::apply_supervisor_startup_hardening()?; + + #[cfg(target_os = "linux")] + let bypass_handle = netns.and_then(|namespace| { + crate::bypass_monitor::spawn( + namespace.name().to_string(), + entrypoint_pid.clone(), + bypass_denial_tx, + bypass_activity_tx, + ) + }); + + #[cfg(target_os = "linux")] + { + let mode = if std::env::var_os("OPENSHELL_REQUIRE_RUNTIME_PID_LIMIT").is_some() { + crate::process::RuntimePidLimitMode::Require + } else { + crate::process::RuntimePidLimitMode::Warn + }; + crate::process::check_runtime_pid_limit(mode)?; + } + + #[cfg(target_os = "linux")] + let boundary_netns_fd = netns + .map(NetworkNamespace::try_clone_ns_fd) + .transpose()? + .flatten() + .map(Arc::new); + #[cfg(not(target_os = "linux"))] + let boundary_netns_fd = None; + + let boundary_runtime = boundary_runtime + .unwrap_or_else(crate::boundary_io::BoundaryRuntimeState::new_exclusive_pid_namespace); + let proxy_url = ssh_proxy_url_for_policy( + policy, + #[cfg(target_os = "linux")] + netns.map(NetworkNamespace::host_ip), + #[cfg(not(target_os = "linux"))] + None, + ); + let user_environment = std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) + .ok() + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_default(); + let port_forward: Arc = + Arc::new(crate::boundary_io::NetnsPortForward::new( + boundary_netns_fd.clone(), + Some(boundary_runtime.clone()), + )); + let boundary_exec: Arc = + Arc::new(crate::boundary_exec::LocalBoundaryExec::new( + policy.clone(), + workspace.owned_root(), + boundary_netns_fd, + proxy_url, + ca_file_paths.clone().map(Arc::new), + provider_credentials, + user_environment, + resolved_process_identity, + enforcement_mode, + boundary_runtime.clone(), + )); + + #[cfg(target_os = "linux")] + let mut handle = ProcessHandle::spawn( + program, + args, + &workspace, + interactive, + policy, + resolved_process_identity, + enforcement_mode, + netns, + ca_file_paths.as_ref(), + &provider_env, + )?; + #[cfg(not(target_os = "linux"))] + let mut handle = ProcessHandle::spawn( + program, + args, + &workspace, + interactive, + policy, + resolved_process_identity, + enforcement_mode, + ca_file_paths.as_ref(), + &provider_env, + )?; + + entrypoint_pid.store(handle.pid(), Ordering::Release); + if let Some(sender) = entrypoint_started_tx { + let _ = sender.send(handle.pid()); + } + let main_session = crate::main_session::MainSession::new(handle.take_io(), handle.pid()); + let (terminal, signal_lock) = handle.signaling_state(); + boundary_runtime + .register_process_group(handle.pid(), terminal.clone(), signal_lock.clone()) + .map_err(|error| miette::miette!(error.to_string()))?; + + ocsf_emit!( + ProcessActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .launch_type(LaunchTypeId::Spawn) + .process(OcsfProcess::new(program, i64::from(handle.pid()))) + .message(format!("Process started: pid={}", handle.pid())) + .build() + ); + + Ok(SpawnedAgent { + instance_id: instance_id.unwrap_or_else(|| uuid::Uuid::new_v4().to_string()), + handle, + timeout_secs, + terminal, + signal_lock, + main_session, + #[cfg(target_os = "linux")] + _bypass_handle: bypass_handle, + boundary_exec, + port_forward, + boundary_runtime, + }) +} + +/// Owned workload process and its live boundary capabilities. +pub struct SpawnedAgent { + instance_id: String, + handle: ProcessHandle, + timeout_secs: u64, + terminal: Arc, + signal_lock: Arc>, + main_session: Arc, + #[cfg(target_os = "linux")] + _bypass_handle: Option>, + boundary_exec: Arc, + port_forward: Arc, + boundary_runtime: Arc, +} + +impl SpawnedAgent { + #[must_use] + pub fn instance_id(&self) -> &str { + &self.instance_id + } + + #[must_use] + pub fn pid(&self) -> u32 { + self.handle.pid() + } + + #[must_use] + pub fn signaler(&self) -> AgentSignaler { + AgentSignaler { + pid: self.handle.pid(), + terminal: self.terminal.clone(), + signal_lock: self.signal_lock.clone(), + } + } + + #[must_use] + pub fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } + + #[must_use] + pub fn port_forward(&self) -> Arc { + self.port_forward.clone() + } + + /// Wait for exit, enforcing the admitted wall-clock timeout. + pub async fn wait(&mut self) -> Result { + let signaler = self.signaler(); + let status = if self.timeout_secs == 0 { + self.handle.wait().await.into_diagnostic()? + } else if let Ok(status) = + tokio::time::timeout(Duration::from_secs(self.timeout_secs), self.handle.wait()).await + { + status.into_diagnostic()? + } else { + let _ = signaler.term(); + tokio::time::sleep(Duration::from_millis(100)).await; + let _ = signaler.kill(); + self.handle.wait().await.into_diagnostic()? + }; + self.boundary_runtime + .unregister_process_group(self.handle.pid(), &self.terminal); + self.boundary_runtime.deactivate(); + let _ = self.main_session.finish(status.code(), false).await; + self.main_session.mark_terminal_reported(); + Ok(status) + } +} + +/// Lock-free process-group signal handle used while another task owns `wait`. +#[derive(Clone)] +pub struct AgentSignaler { + pid: u32, + terminal: Arc, + signal_lock: Arc>, +} + +#[cfg(unix)] +impl AgentSignaler { + fn deliver(&self, signal: nix::sys::signal::Signal) -> Result<()> { + let _guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("agent has exited")); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + nix::sys::signal::killpg(nix::unistd::Pid::from_raw(pid), signal).into_diagnostic() + } + + pub fn term(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGTERM) + } + + pub fn kill(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGKILL) + } + + pub fn interrupt(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGINT) + } + + pub fn hangup(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGHUP) + } +} + +fn ssh_proxy_url_for_policy( + policy: &SandboxPolicy, + netns_proxy_host: Option, +) -> Option { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return None; + } + let proxy = policy.network.proxy.as_ref()?; + netns_proxy_host.map_or_else( + || proxy.http_addr.map(|address| format!("http://{address}")), + |host| { + let port = proxy.http_addr.map_or(3128, |address| address.port()); + Some(format!("http://{host}:{port}")) + }, + ) +} + +/// Report and finalize the canonical process exit from control mode. +pub async fn report_main_process_completion( + endpoint: &str, + sandbox_id: &str, + instance_id: &str, + exit_code: i32, +) { + let mut delay = Duration::from_millis(250); + loop { + match crate::supervisor_session::report_main_process_exit( + endpoint, + sandbox_id, + instance_id, + exit_code, + ) + .await + { + Ok(()) => break, + Err(error) => { + tracing::warn!(%error, "main-process exit report failed; retrying"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(2)); + } + } + } + delay = Duration::from_millis(250); + loop { + match crate::supervisor_session::finalize_main_process_exit( + endpoint, + sandbox_id, + instance_id, + ) + .await + { + Ok(()) => break, + Err(error) => { + tracing::warn!(%error, "main-process finalization failed; retrying"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(2)); + } + } + } +} diff --git a/crates/openshell-supervisor-process/src/lib.rs b/crates/openshell-supervisor-process/src/lib.rs index ee6bedeb22..ba4ce12acd 100644 --- a/crates/openshell-supervisor-process/src/lib.rs +++ b/crates/openshell-supervisor-process/src/lib.rs @@ -12,6 +12,7 @@ pub mod boundary_exec; pub mod boundary_io; pub mod child_env; pub mod debug_rpc; +pub mod delegated; #[cfg(unix)] pub mod identity; pub mod log_push; diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 2b4ea554ed..186d11e721 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -11,9 +11,12 @@ mod nft_ruleset; use miette::{IntoDiagnostic, Result}; use std::net::IpAddr; +use std::os::fd::AsRawFd as _; +use std::os::fd::{BorrowedFd, OwnedFd}; use std::os::unix::io::RawFd; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Command; +use std::sync::OnceLock; use tracing::{debug, warn}; use uuid::Uuid; @@ -26,13 +29,80 @@ const SANDBOX_IP_SUFFIX: u8 = 2; /// this listener before the bypass fence runs. pub const POLICY_DNS_PORT: u16 = 15_053; pub const TRANSPARENT_TCP_PORT: u16 = 15_001; -const IP_SEARCH_PATHS: &[&str] = &["/usr/sbin/ip", "/sbin/ip", "/usr/bin/ip", "/bin/ip"]; -const NSENTER_SEARCH_PATHS: &[&str] = &[ - "/usr/bin/nsenter", - "/bin/nsenter", - "/usr/sbin/nsenter", - "/sbin/nsenter", -]; +const IP_SEARCH_PATHS: &[&str] = &["usr/sbin/ip", "sbin/ip", "usr/bin/ip", "bin/ip"]; +static TRUSTED_RUNTIME_ROOT: OnceLock = OnceLock::new(); + +/// Pin the driver-owned helper runtime used for conformant namespace setup. +/// +/// VM guest leaves call this before starting any control or workload threads +/// because their executable may be launched through a dynamic loader. In that +/// case `/proc/self/exe` identifies the loader rather than the supervisor +/// binary, so the default executable-relative lookup is not authoritative. +/// +/// # Errors +/// +/// Returns an error for a relative path or if another root was already pinned. +pub fn configure_trusted_runtime_root(root: PathBuf) -> Result<()> { + if !root.is_absolute() { + return Err(miette::miette!( + "trusted supervisor helper runtime root must be absolute" + )); + } + TRUSTED_RUNTIME_ROOT.set(root).map_err(|configured| { + miette::miette!( + "trusted supervisor helper runtime root is already configured as {}", + configured.display() + ) + }) +} + +#[derive(Clone, Debug)] +struct TrustedHelper { + executable: PathBuf, + loader: Option, + library_path: String, + xtables_path: PathBuf, +} + +impl TrustedHelper { + fn command(&self) -> Command { + self.loader.as_ref().map_or_else( + || Command::new(&self.executable), + |loader| { + let mut command = Command::new(loader); + command + .env_clear() + .env("XTABLES_LIBDIR", &self.xtables_path) + .arg("--library-path") + .arg(&self.library_path) + .arg(&self.executable); + command + }, + ) + } + + fn tokio_command(&self) -> tokio::process::Command { + self.loader.as_ref().map_or_else( + || tokio::process::Command::new(&self.executable), + |loader| { + let mut command = tokio::process::Command::new(loader); + command + .env_clear() + .env("XTABLES_LIBDIR", &self.xtables_path) + .arg("--library-path") + .arg(&self.library_path) + .arg(&self.executable); + command + }, + ) + } +} + +#[derive(Clone, Copy, Debug)] +enum HelperSource { + LegacyWorkloadImage, + TrustedSupervisorRuntime, +} /// Handle to a network namespace with veth pair. /// @@ -44,13 +114,24 @@ pub struct NetworkNamespace { /// Host-side veth interface name veth_host: String, /// Sandbox-side veth interface name (inside namespace, used only during setup) - _veth_sandbox: String, + #[allow(dead_code)] + veth_sandbox: String, /// Host-side IP address (proxy binds here) host_ip: IpAddr, /// Sandbox-side IP address sandbox_ip: IpAddr, /// File descriptor for the namespace (for setns) ns_fd: Option, + helper_source: HelperSource, +} + +/// Cloneable coordinates for checking a live ceiling without retaining the +/// namespace fd or delaying namespace cleanup. +#[derive(Clone, Debug)] +pub struct EgressCeilingVerifier { + namespace: String, + host_ip: IpAddr, + helper_source: HelperSource, } impl NetworkNamespace { @@ -66,6 +147,14 @@ impl NetworkNamespace { /// /// Returns an error if namespace creation or network setup fails. pub fn create() -> Result { + Self::create_with_helper_source(HelperSource::LegacyWorkloadImage) + } + + fn create_conformant() -> Result { + Self::create_with_helper_source(HelperSource::TrustedSupervisorRuntime) + } + + fn create_with_helper_source(helper_source: HelperSource) -> Result { let id = Uuid::new_v4(); let short_id = &id.to_string()[..8]; let name = format!("sandbox-{short_id}"); @@ -89,84 +178,101 @@ impl NetworkNamespace { ); // Create the namespace - run_ip(&["netns", "add", &name])?; + run_ip(helper_source, &["netns", "add", &name])?; // Create veth pair - if let Err(e) = run_ip(&[ - "link", - "add", - &veth_host, - "type", - "veth", - "peer", - "name", - &veth_sandbox, - ]) { + if let Err(e) = run_ip( + helper_source, + &[ + "link", + "add", + &veth_host, + "type", + "veth", + "peer", + "name", + &veth_sandbox, + ], + ) { // Cleanup namespace on failure - let _ = run_ip(&["netns", "delete", &name]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Move sandbox veth into namespace - if let Err(e) = run_ip(&["link", "set", &veth_sandbox, "netns", &name]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip( + helper_source, + &["link", "set", &veth_sandbox, "netns", &name], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Configure host side let host_cidr = format!("{host_ip}/24"); - if let Err(e) = run_ip(&["addr", "add", &host_cidr, "dev", &veth_host]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip( + helper_source, + &["addr", "add", &host_cidr, "dev", &veth_host], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } - if let Err(e) = run_ip(&["link", "set", &veth_host, "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip(helper_source, &["link", "set", &veth_host, "up"]) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Configure sandbox side (inside namespace) let sandbox_cidr = format!("{sandbox_ip}/24"); - if let Err(e) = run_ip_netns(&name, &["addr", "add", &sandbox_cidr, "dev", &veth_sandbox]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns( + helper_source, + &name, + &["addr", "add", &sandbox_cidr, "dev", &veth_sandbox], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } - if let Err(e) = run_ip_netns(&name, &["link", "set", &veth_sandbox, "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns(helper_source, &name, &["link", "set", &veth_sandbox, "up"]) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Bring up loopback in namespace - if let Err(e) = run_ip_netns(&name, &["link", "set", "lo", "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns(helper_source, &name, &["link", "set", "lo", "up"]) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Add default route via host let host_ip_str = host_ip.to_string(); - if let Err(e) = run_ip_netns(&name, &["route", "add", "default", "via", &host_ip_str]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns( + helper_source, + &name, + &["route", "add", "default", "via", &host_ip_str], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Open the namespace file descriptor for later use with setns - let ns_path = openshell_core::container_paths::netns_path(&name); + let ns_path = format!("/var/run/netns/{name}"); let ns_fd = match nix::fcntl::open( - ns_path.as_path(), + ns_path.as_str(), nix::fcntl::OFlag::O_RDONLY, nix::sys::stat::Mode::empty(), ) { Ok(fd) => Some(fd), Err(e) => { - warn!(error = %e, "Failed to open namespace fd, will use nsenter fallback"); + warn!(error = %e, "Failed to retain network namespace fd"); None } }; @@ -185,10 +291,11 @@ impl NetworkNamespace { Ok(Self { name, veth_host, - _veth_sandbox: veth_sandbox, + veth_sandbox, host_ip, sandbox_ip, ns_fd, + helper_source, }) } @@ -249,25 +356,21 @@ impl NetworkNamespace { self.ns_fd } - /// Install nftables rules for bypass detection inside the namespace. - /// - /// Sets up OUTPUT chain rules that: - /// 1. ACCEPT traffic destined for the proxy (`host_ip:proxy_port`) - /// 2. ACCEPT loopback traffic - /// 3. ACCEPT established/related connections (response packets) - /// 4. LOG + REJECT all other TCP/UDP traffic (bypass attempts) - /// - /// This provides two benefits: - /// - **Fast-fail UX**: applications get immediate ECONNREFUSED instead of - /// a 30-second timeout when they bypass the proxy - /// - **Diagnostics**: nftables LOG entries are picked up by the bypass - /// monitor to emit structured tracing events - /// - /// Degrades gracefully if `nft` is not available — the namespace - /// still provides isolation via routing, just without fast-fail and - /// diagnostic logging. + /// Duplicate the namespace descriptor for a retained runtime handle. + pub fn try_clone_ns_fd(&self) -> Result> { + self.ns_fd + .map(|fd| { + // SAFETY: `NetworkNamespace` owns `fd` for at least this call. + #[allow(unsafe_code)] + let borrowed = unsafe { BorrowedFd::borrow_raw(fd) }; + borrowed.try_clone_to_owned().into_diagnostic() + }) + .transpose() + } + + /// Install the legacy best-effort nftables bypass-detection rules. pub fn install_bypass_rules(&self, proxy_port: u16) -> Result<()> { - let Some(nft_path) = find_nft() else { + let Some(nft_path) = find_nft(self.helper_source) else { openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Medium) @@ -281,33 +384,25 @@ impl NetworkNamespace { ); return Ok(()); }; - - let host_ip_str = self.host_ip.to_string(); + let host_ip = self.host_ip.to_string(); let log_prefix = format!("openshell:bypass:{}:", &self.name); - - // The kernel's nf_log_syslog module suppresses log output from - // non-init network namespaces by default. Enable it so the bypass - // monitor can see log entries from the sandbox namespace. enable_nf_log_all_netns(); - let commands = - nft_ruleset::generate_bypass_commands(&host_ip_str, proxy_port, Some(&log_prefix)); - - if let Err(e) = run_nft_commands_netns(&self.name, &nft_path, &commands) { + nft_ruleset::generate_bypass_commands(&host_ip, proxy_port, Some(&log_prefix)); + if let Err(error) = run_nft_commands_netns(&self.name, &nft_path, &commands) { openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Medium) .status(openshell_ocsf::StatusId::Failure) .state(openshell_ocsf::StateId::Disabled, "failed") .message(format!( - "Failed to install bypass detection rules [ns:{}]: {e}", + "Failed to install bypass detection rules [ns:{}]: {error}", self.name )) .build() ); - return Err(e); + return Err(error); } - openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Informational) @@ -319,7 +414,6 @@ impl NetworkNamespace { )) .build() ); - Ok(()) } @@ -338,10 +432,11 @@ impl NetworkNamespace { // kernel reaches the nft OUTPUT hook; REDIRECT then reroutes it to // the local transparent listener. run_ip_netns( + self.helper_source, &self.name, &["-6", "route", "replace", synthetic_ipv6_cidr, "dev", "lo"], )?; - let nft_path = find_nft().ok_or_else(|| { + let nft_path = find_nft(self.helper_source).ok_or_else(|| { miette::miette!( "trusted nft helper not found; policy DNS and transparent TCP require nftables" ) @@ -386,8 +481,11 @@ impl NetworkNamespace { .into_diagnostic()?, ]; for family in ["-4", "-6"] { - let routes = - run_ip_netns_output(&self.name, &[family, "route", "show", "table", "all"])?; + let routes = run_ip_netns_output( + self.helper_source, + &self.name, + &[family, "route", "show", "table", "all"], + )?; if let Some((route, pool)) = first_route_overlap(&routes, &reserved) { return Err(miette::miette!( "synthetic address pool {pool} overlaps workload route {route}; refusing to enable policy DNS" @@ -510,6 +608,101 @@ impl NetworkNamespace { )) } + /// Install the RFC 0012 default-deny egress ceiling inside the namespace. + /// + /// Sets up OUTPUT chain rules that: + /// 1. ACCEPT traffic destined for the proxy (`host_ip:proxy_port`) + /// 2. ACCEPT loopback traffic + /// 3. LOG + REJECT TCP/UDP bypass attempts and DROP every other packet + /// + /// This provides two benefits: + /// - **Fast-fail UX**: applications get immediate ECONNREFUSED instead of + /// a 30-second timeout when they bypass the proxy + /// - **Diagnostics**: nftables LOG entries are picked up by the bypass + /// monitor to emit structured tracing events + /// + /// Missing nftables support is fatal: without the default-deny ceiling the + /// backend cannot confirm that all workload egress reaches mediation. + pub fn install_egress_ceiling(&self, proxy_port: u16) -> Result<()> { + let Some(nft_path) = find_nft(self.helper_source) else { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::High) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Disabled, "unavailable") + .message(format!( + "nft not found; refusing to establish the egress ceiling [ns:{}]", + self.name + )) + .build() + ); + return Err(miette::miette!( + "nft not found; cannot establish default-deny egress ceiling" + )); + }; + + let host_ip_str = self.host_ip.to_string(); + let log_prefix = format!("openshell:bypass:{}:", &self.name); + + // The kernel's nf_log_syslog module suppresses log output from + // non-init network namespaces by default. Enable it so the bypass + // monitor can see log entries from the sandbox namespace. + enable_nf_log_all_netns(); + + let commands = nft_ruleset::generate_egress_ceiling_commands( + &host_ip_str, + proxy_port, + Some(&log_prefix), + ); + + if let Err(e) = run_nft_commands_netns(&self.name, &nft_path, &commands) { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::High) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Disabled, "failed") + .message(format!( + "Failed to establish egress ceiling [ns:{}]: {e}", + self.name + )) + .build() + ); + return Err(e); + } + + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "installed") + .message(format!( + "Default-deny egress ceiling established [ns:{}]", + self.name + )) + .build() + ); + + Ok(()) + } + + /// Verify the live default-deny egress ceiling installed for this boundary. + /// + /// This reads the ruleset back from the kernel rather than treating a + /// successful installation attempt as proof that standing enforcement is + /// still present. + pub fn verify_egress_ceiling(&self, proxy_port: u16) -> Result<()> { + self.egress_ceiling_verifier().verify(proxy_port) + } + + #[must_use] + pub fn egress_ceiling_verifier(&self) -> EgressCeilingVerifier { + EgressCeilingVerifier { + namespace: self.name.clone(), + host_ip: self.host_ip, + helper_source: self.helper_source, + } + } + /// Bind a TCP listener inside this network namespace on a dedicated thread. /// /// Spawns a short-lived OS thread that enters the namespace via `setns`, @@ -548,6 +741,158 @@ impl NetworkNamespace { } } +impl EgressCeilingVerifier { + fn nft_helper(&self) -> Result { + find_nft(self.helper_source) + .ok_or_else(|| miette::miette!("nft not found; cannot verify egress ceiling")) + } + + fn verify(&self, proxy_port: u16) -> Result<()> { + let nft = self.nft_helper()?; + let output = trusted_command_in_netns(&nft, &self.namespace)? + .args(["-j", "list", "chain", "inet", "openshell_bypass", "output"]) + .output() + .into_diagnostic()?; + self.validate_output(proxy_port, &output) + } + + /// Run a verifier helper with a hard deadline. Dropping the timed-out + /// future kills the child, so a stuck `nft` cannot retain the + /// namespace or suspend enforcement-loss detection indefinitely. + pub async fn verify_bounded( + &self, + proxy_port: u16, + timeout: std::time::Duration, + ) -> Result<()> { + let nft = self.nft_helper()?; + let mut command = trusted_tokio_command_in_netns(&nft, &self.namespace)?; + command.kill_on_drop(true).args([ + "-j", + "list", + "chain", + "inet", + "openshell_bypass", + "output", + ]); + let output = tokio::time::timeout(timeout, command.output()) + .await + .map_err(|_| miette::miette!("egress ceiling verification timed out"))? + .into_diagnostic()?; + self.validate_output(proxy_port, &output) + } + + fn validate_output(&self, proxy_port: u16, output: &std::process::Output) -> Result<()> { + if !output.status.success() { + return Err(miette::miette!( + "could not read back egress ceiling in netns {}: {}", + self.namespace, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + verify_egress_ceiling_json(&output.stdout, &self.host_ip.to_string(), proxy_port) + } +} + +fn verify_egress_ceiling_json(json: &[u8], host_ip: &str, proxy_port: u16) -> Result<()> { + let document: serde_json::Value = serde_json::from_slice(json).into_diagnostic()?; + let objects = document + .get("nftables") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| miette::miette!("nft response has no nftables object list"))?; + + let chain_is_default_deny = objects.iter().any(|object| { + let Some(chain) = object.get("chain") else { + return false; + }; + chain.get("family").and_then(serde_json::Value::as_str) == Some("inet") + && chain.get("table").and_then(serde_json::Value::as_str) == Some("openshell_bypass") + && chain.get("name").and_then(serde_json::Value::as_str) == Some("output") + && chain.get("type").and_then(serde_json::Value::as_str) == Some("filter") + && chain.get("hook").and_then(serde_json::Value::as_str) == Some("output") + && chain.get("prio").and_then(serde_json::Value::as_i64) == Some(0) + && chain.get("policy").and_then(serde_json::Value::as_str) == Some("drop") + }); + if !chain_is_default_deny { + return Err(miette::miette!( + "egress ceiling output chain is absent or not policy drop" + )); + } + + let output_rules: Vec<&serde_json::Value> = objects + .iter() + .filter_map(|object| object.get("rule")) + .collect(); + for rule in &output_rules { + let expressions = rule + .get("expr") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| miette::miette!("egress ceiling rule has no expression list"))?; + for expression in expressions { + let keys = expression + .as_object() + .ok_or_else(|| miette::miette!("egress ceiling contains a malformed expression"))?; + if keys.len() != 1 + || !keys.keys().all(|key| { + matches!( + key.as_str(), + "match" | "counter" | "limit" | "log" | "reject" | "drop" | "accept" + ) + }) + { + return Err(miette::miette!( + "egress ceiling contains an unsupported or redirecting expression" + )); + } + } + } + let accept_rules: Vec<&serde_json::Value> = output_rules + .into_iter() + .filter(|rule| { + rule.get("family").and_then(serde_json::Value::as_str) == Some("inet") + && rule.get("table").and_then(serde_json::Value::as_str) == Some("openshell_bypass") + && rule.get("chain").and_then(serde_json::Value::as_str) == Some("output") + }) + .filter(|rule| { + rule.get("expr") + .and_then(serde_json::Value::as_array) + .is_some_and(|expressions| { + expressions + .iter() + .any(|expression| expression == &serde_json::json!({"accept": null})) + }) + }) + .collect(); + let proxy_expressions = serde_json::json!([ + {"match":{"op":"==","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":host_ip}}, + {"match":{"op":"==","left":{"payload":{"protocol":"tcp","field":"dport"}},"right":proxy_port}}, + {"accept":null} + ]); + let loopback_expressions = serde_json::json!([ + {"match":{"op":"==","left":{"meta":{"key":"oifname"}},"right":"lo"}}, + {"accept":null} + ]); + let mut proxy_allowed = false; + let mut loopback_allowed = false; + for rule in accept_rules { + let expressions = rule.get("expr").expect("accept rule has expressions"); + if expressions == &proxy_expressions { + proxy_allowed = true; + } else if expressions == &loopback_expressions { + loopback_allowed = true; + } else { + return Err(miette::miette!( + "egress ceiling contains an unexpected accept rule: {expressions}" + )); + } + } + if !proxy_allowed || !loopback_allowed { + return Err(miette::miette!( + "egress ceiling is missing the proxy or loopback accept rule" + )); + } + Ok(()) +} + impl Drop for NetworkNamespace { fn drop(&mut self) { debug!(namespace = %self.name, "Cleaning up network namespace"); @@ -558,7 +903,9 @@ impl Drop for NetworkNamespace { } // Delete the host-side veth (this also removes the peer) - if let Err(e) = run_ip(&["link", "delete", &self.veth_host]) { + let mut cleanup_failed = false; + if let Err(e) = run_ip(self.helper_source, &["link", "delete", &self.veth_host]) { + cleanup_failed = true; warn!( error = %e, veth = %self.veth_host, @@ -567,7 +914,8 @@ impl Drop for NetworkNamespace { } // Delete the namespace - if let Err(e) = run_ip(&["netns", "delete", &self.name]) { + if let Err(e) = run_ip(self.helper_source, &["netns", "delete", &self.name]) { + cleanup_failed = true; warn!( error = %e, namespace = %self.name, @@ -575,14 +923,32 @@ impl Drop for NetworkNamespace { ); } - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Disabled, "cleaned_up") - .message(format!("Network namespace cleaned up [ns:{}]", self.name)) - .build() - ); + let event = openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(if cleanup_failed { + openshell_ocsf::SeverityId::High + } else { + openshell_ocsf::SeverityId::Informational + }) + .status(if cleanup_failed { + openshell_ocsf::StatusId::Failure + } else { + openshell_ocsf::StatusId::Success + }) + .state( + openshell_ocsf::StateId::Disabled, + if cleanup_failed { + "cleanup_failed" + } else { + "cleaned_up" + }, + ) + .message(if cleanup_failed { + format!("Network namespace cleanup incomplete [ns:{}]", self.name) + } else { + format!("Network namespace cleaned up [ns:{}]", self.name) + }) + .build(); + openshell_ocsf::ocsf_emit!(event); } } @@ -597,10 +963,25 @@ impl Drop for NetworkNamespace { /// /// Returns an error if proxy mode is requested but the namespace cannot be /// created (e.g., missing `CAP_NET_ADMIN` / `CAP_SYS_ADMIN` or `iproute2`). -/// Failure to install nftables bypass-detection rules is non-fatal and is -/// reported via OCSF instead. +/// Legacy bypass-rule installation remains best-effort for compatibility. pub fn create_netns_for_proxy( policy: &openshell_core::policy::SandboxPolicy, +) -> Result> { + create_netns(policy, false) +} + +/// Create a proxy namespace whose nftables policy is a mandatory RFC 0012 +/// default-deny ceiling. Unlike the legacy helper, any installation failure +/// aborts boundary establishment. +pub fn create_conformant_netns_for_proxy( + policy: &openshell_core::policy::SandboxPolicy, +) -> Result> { + create_netns(policy, true) +} + +fn create_netns( + policy: &openshell_core::policy::SandboxPolicy, + require_egress_ceiling: bool, ) -> Result> { use openshell_core::policy::NetworkMode; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; @@ -608,7 +989,12 @@ pub fn create_netns_for_proxy( if !matches!(policy.network.mode, NetworkMode::Proxy) { return Ok(None); } - match NetworkNamespace::create() { + let namespace = if require_egress_ceiling { + NetworkNamespace::create_conformant() + } else { + NetworkNamespace::create() + }; + match namespace { Ok(ns) => { let proxy_port = policy .network @@ -616,14 +1002,26 @@ pub fn create_netns_for_proxy( .as_ref() .and_then(|p| p.http_addr) .map_or(3128, |addr| addr.port()); - if let Err(e) = ns.install_bypass_rules(proxy_port) { + if require_egress_ceiling { + ns.install_egress_ceiling(proxy_port).map_err(|error| { + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "failed") + .message(format!("Failed to establish egress ceiling: {error}")) + .build() + ); + error + })?; + } else if let Err(error) = ns.install_bypass_rules(proxy_port) { ocsf_emit!( ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(SeverityId::Medium) .status(StatusId::Failure) .state(StateId::Disabled, "degraded") .message(format!( - "Failed to install bypass detection rules (non-fatal): {e}" + "Failed to install bypass detection rules (non-fatal): {error}" )) .build() ); @@ -666,7 +1064,7 @@ pub fn install_sidecar_bypass_rules(proxy_uid: u32) -> Result<()> { } fn install_sidecar_nft_bypass_rules(proxy_uid: u32) -> Result<()> { - let nft_cmd = find_nft().ok_or_else(|| { + let nft_cmd = find_nft(HelperSource::TrustedSupervisorRuntime).ok_or_else(|| { miette::miette!( "trusted nft helper not found; sidecar network enforcement requires nftables" ) @@ -680,14 +1078,14 @@ const SIDECAR_IPTABLES_CHAIN: &str = "OPENSHELL_SIDECAR_BYPASS"; const PROC_NET_IF_INET6_PATH: &str = "/proc/net/if_inet6"; fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { - let ipv4_filter_tool = find_iptables_legacy().ok_or_else(|| { + let ipv4_filter_tool = find_iptables_legacy(HelperSource::TrustedSupervisorRuntime).ok_or_else(|| { miette::miette!( "trusted iptables-legacy helper not found; sidecar network enforcement fallback unavailable" ) })?; let ipv6_fence_tool = if current_namespace_has_non_loopback_ipv6()? { - Some(find_ip6tables_legacy().ok_or_else(|| { + Some(find_ip6tables_legacy(HelperSource::TrustedSupervisorRuntime).ok_or_else(|| { miette::miette!( "trusted ip6tables-legacy helper not found; sidecar network enforcement fallback cannot fence IPv6" ) @@ -699,17 +1097,14 @@ fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { None }; - cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_deref()); + cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_ref()); if let Err(e) = install_sidecar_iptables_legacy_family_rules( &ipv4_filter_tool, proxy_uid, "icmp-port-unreachable", ) { - cleanup_sidecar_iptables_legacy_rule_families( - &ipv4_filter_tool, - ipv6_fence_tool.as_deref(), - ); + cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_ref()); return Err(e); } @@ -746,7 +1141,7 @@ fn has_non_loopback_ipv6_interface(content: &str) -> bool { } fn install_sidecar_iptables_legacy_family_rules( - cmd: &str, + cmd: &TrustedHelper, proxy_uid: u32, udp_reject_with: &str, ) -> Result<()> { @@ -807,7 +1202,7 @@ fn install_sidecar_iptables_legacy_family_rules( Ok(()) } -fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { +fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &TrustedHelper) { while run_iptables_legacy_current_namespace( iptables_cmd, &["-D", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], @@ -818,28 +1213,70 @@ fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-X", SIDECAR_IPTABLES_CHAIN]); } -fn cleanup_sidecar_iptables_legacy_rule_families(ipv4_cmd: &str, ipv6_cmd: Option<&str>) { +fn cleanup_sidecar_iptables_legacy_rule_families( + ipv4_cmd: &TrustedHelper, + ipv6_cmd: Option<&TrustedHelper>, +) { cleanup_sidecar_iptables_legacy_rules(ipv4_cmd); if let Some(ipv6_cmd) = ipv6_cmd { cleanup_sidecar_iptables_legacy_rules(ipv6_cmd); } } +#[allow(unsafe_code)] +fn trusted_command_in_netns(helper: &TrustedHelper, netns: &str) -> Result { + use std::os::unix::process::CommandExt as _; + + let namespace = std::fs::File::open(format!("/var/run/netns/{netns}")).into_diagnostic()?; + let mut command = helper.command(); + // SAFETY: `setns` is async-signal-safe and the captured file remains open + // in the child until this pre-exec hook completes. + unsafe { + command.pre_exec(move || { + if libc::setns(namespace.as_raw_fd(), libc::CLONE_NEWNET) == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + Ok(command) +} + +#[allow(unsafe_code)] +fn trusted_tokio_command_in_netns( + helper: &TrustedHelper, + netns: &str, +) -> Result { + let namespace = std::fs::File::open(format!("/var/run/netns/{netns}")).into_diagnostic()?; + let mut command = helper.tokio_command(); + // SAFETY: `setns` is async-signal-safe and the captured file remains open + // in the child until this pre-exec hook completes. + unsafe { + command.pre_exec(move || { + if libc::setns(namespace.as_raw_fd(), libc::CLONE_NEWNET) == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + Ok(command) +} + /// Run an `ip` command on the host. -fn run_ip(args: &[&str]) -> Result<()> { - let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; +fn run_ip(source: HelperSource, args: &[&str]) -> Result<()> { + let ip = find_binary(source, "ip", IP_SEARCH_PATHS)?; - debug!(command = %format!("{ip_path} {}", args.join(" ")), "Running ip command"); + debug!(command = %format!("{} {}", ip.executable.display(), args.join(" ")), "Running ip command"); - let output = Command::new(ip_path) - .args(args) - .output() - .into_diagnostic()?; + let output = ip.command().args(args).output().into_diagnostic()?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "{ip_path} {} failed: {}", + "{} {} failed: {}", + ip.executable.display(), args.join(" "), stderr.trim() )); @@ -848,13 +1285,17 @@ fn run_ip(args: &[&str]) -> Result<()> { Ok(()) } -fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> Result<()> { +fn run_iptables_legacy_current_namespace( + iptables_cmd: &TrustedHelper, + args: &[&str], +) -> Result<()> { debug!( - command = %format!("{iptables_cmd} {}", args.join(" ")), + command = %format!("{} {}", iptables_cmd.executable.display(), args.join(" ")), "Running iptables-legacy sidecar command" ); - let output = Command::new(iptables_cmd) + let output = iptables_cmd + .command() .args(args) .output() .into_diagnostic()?; @@ -862,7 +1303,8 @@ fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> R if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "{iptables_cmd} {} failed: {}", + "{} {} failed: {}", + iptables_cmd.executable.display(), args.join(" "), stderr.trim() )); @@ -880,14 +1322,15 @@ fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> R /// Commands marked as non-required are allowed to fail with a warning. /// Required commands that fail abort the sequence immediately. fn run_nft_commands_current_namespace( - nft_cmd: &str, + nft_cmd: &TrustedHelper, commands: &[nft_ruleset::NftCommand], ) -> Result<()> { for cmd in commands { let args_str = cmd.args.join(" "); - debug!(command = %format!("{nft_cmd} {args_str}"), "Running nft command"); + debug!(command = %format!("{} {args_str}", nft_cmd.executable.display()), "Running nft command"); - let output = Command::new(nft_cmd) + let output = nft_cmd + .command() .args(&cmd.args) .output() .into_diagnostic()?; @@ -896,7 +1339,8 @@ fn run_nft_commands_current_namespace( let stderr = String::from_utf8_lossy(&output.stderr); if cmd.required { return Err(miette::miette!( - "{nft_cmd} {args_str} failed: {}", + "{} {args_str} failed: {}", + nft_cmd.executable.display(), stderr.trim() )); } @@ -910,44 +1354,32 @@ fn run_nft_commands_current_namespace( Ok(()) } -/// Run an `ip` command inside a network namespace via `nsenter --net=`. +/// Run an `ip` command inside a network namespace. /// -/// We use `nsenter` instead of `ip netns exec` because `ip netns exec` -/// remounts `/sys` to reflect the target namespace's sysfs entries. That -/// sysfs remount requires real `CAP_SYS_ADMIN` in the host user namespace, -/// which is unavailable in rootless container runtimes (e.g. rootless -/// Podman). `nsenter --net=` enters only the network namespace without -/// changing the mount namespace, avoiding the sysfs remount entirely. -/// The supervisor's operations (addr add, link set, route add) are all -/// netlink-based and do not need sysfs access. -fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { - run_ip_netns_output(netns, args).map(|_| ()) +/// The child enters only the network namespace before exec. This avoids both +/// `ip netns exec`'s sysfs remount and a separate `nsenter` helper. +fn run_ip_netns(source: HelperSource, netns: &str, args: &[&str]) -> Result<()> { + run_ip_netns_output(source, netns, args).map(|_| ()) } -fn run_ip_netns_output(netns: &str, args: &[&str]) -> Result { - let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; - let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; - let ns_path = openshell_core::container_paths::netns_path(netns); - let net_flag = format!("--net={}", ns_path.display()); - - let mut full_args = vec![net_flag.as_str(), "--", ip_path]; - full_args.extend(args); +fn run_ip_netns_output(source: HelperSource, netns: &str, args: &[&str]) -> Result { + let ip = find_binary(source, "ip", IP_SEARCH_PATHS)?; debug!( - command = %format!("{nsenter_path} {}", full_args.join(" ")), - "Running ip in namespace via nsenter" + command = %format!("{} {}", ip.executable.display(), args.join(" ")), + "Running ip in namespace" ); - let output = Command::new(nsenter_path) - .args(&full_args) + let output = trusted_command_in_netns(&ip, netns)? + .args(args) .output() .into_diagnostic()?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "{nsenter_path} --net={} {ip_path} {} failed: {}", - ns_path.display(), + "{} {} failed in netns {netns}: {}", + ip.executable.display(), args.join(" "), stderr.trim() )); @@ -980,32 +1412,25 @@ fn first_route_overlap( }) } -/// Run a sequence of nft commands inside a network namespace via `nsenter --net=`. +/// Run a sequence of nft commands inside a network namespace. /// /// Each command is executed as a separate invocation to avoid atomic batch /// rollback. See [`run_nft_commands_current_namespace`] for rationale. fn run_nft_commands_netns( netns: &str, - nft_cmd: &str, + nft_cmd: &TrustedHelper, commands: &[nft_ruleset::NftCommand], ) -> Result<()> { - let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; - let ns_path = openshell_core::container_paths::netns_path(netns); - let net_flag = format!("--net={}", ns_path.display()); - for cmd in commands { let args_str = cmd.args.join(" "); debug!( - command = %format!("{nsenter_path} {net_flag} -- {nft_cmd} {args_str}"), + command = %format!("{} {args_str}", nft_cmd.executable.display()), "Running nft command in namespace" ); - let mut full_args = vec![net_flag.as_str(), "--", nft_cmd]; let arg_refs: Vec<&str> = cmd.args.iter().map(String::as_str).collect(); - full_args.extend(&arg_refs); - - let output = Command::new(nsenter_path) - .args(&full_args) + let output = trusted_command_in_netns(nft_cmd, netns)? + .args(&arg_refs) .output() .into_diagnostic()?; @@ -1055,109 +1480,282 @@ fn enable_nf_log_all_netns() { } } -/// Well-known paths where nft may be installed. -const NFT_SEARCH_PATHS: &[&str] = &["/usr/sbin/nft", "/sbin/nft", "/usr/bin/nft"]; +/// Paths within the driver-controlled supervisor runtime. +const NFT_SEARCH_PATHS: &[&str] = &["usr/sbin/nft", "sbin/nft", "usr/bin/nft"]; const IPTABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ - "/usr/sbin/iptables-legacy", - "/sbin/iptables-legacy", - "/usr/bin/iptables-legacy", + "usr/sbin/iptables-legacy", + "sbin/iptables-legacy", + "usr/bin/iptables-legacy", ]; const IP6TABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ - "/usr/sbin/ip6tables-legacy", - "/sbin/ip6tables-legacy", - "/usr/bin/ip6tables-legacy", + "usr/sbin/ip6tables-legacy", + "sbin/ip6tables-legacy", + "usr/bin/ip6tables-legacy", ]; -fn find_trusted_binary<'a>(name: &str, paths: &'a [&str]) -> Result<&'a str> { - paths +fn trusted_runtime_root() -> Result { + #[cfg(test)] + if let Some(root) = std::env::var_os("OPENSHELL_TEST_TRUSTED_RUNTIME_ROOT") { + return Ok(PathBuf::from(root)); + } + if let Some(root) = TRUSTED_RUNTIME_ROOT.get() { + return Ok(root.clone()); + } + let executable = std::env::current_exe().into_diagnostic()?; + let parent = executable + .parent() + .ok_or_else(|| miette::miette!("supervisor executable has no parent directory"))?; + Ok(parent.join("openshell-runtime")) +} + +fn find_binary(source: HelperSource, name: &str, paths: &[&str]) -> Result { + match source { + HelperSource::LegacyWorkloadImage => find_legacy_binary(name, paths), + HelperSource::TrustedSupervisorRuntime => find_trusted_binary(name, paths), + } +} + +fn find_legacy_binary(name: &str, paths: &[&str]) -> Result { + use std::os::unix::fs::MetadataExt as _; + + let trusted_uid = nix::unistd::geteuid().as_raw(); + let executable = paths + .iter() + .map(|path| Path::new("/").join(path)) + .find_map(|path| { + let resolved = path.canonicalize().ok()?; + let metadata = resolved.metadata().ok()?; + (metadata.is_file() + && metadata.uid() == trusted_uid + && metadata.mode() & 0o111 != 0 + && metadata.mode() & 0o022 == 0) + .then_some(resolved) + }) + .ok_or_else(|| { + miette::miette!( + "{name} helper not found in legacy workload image; checked {}", + paths.join(", ") + ) + })?; + Ok(TrustedHelper { + executable, + loader: None, + library_path: String::new(), + xtables_path: PathBuf::new(), + }) +} + +fn find_trusted_binary(name: &str, paths: &[&str]) -> Result { + find_trusted_binary_in(&trusted_runtime_root()?, name, paths) +} + +fn find_trusted_binary_in(root: &Path, name: &str, paths: &[&str]) -> Result { + use std::os::unix::fs::MetadataExt; + + let trusted_uid = nix::unistd::geteuid().as_raw(); + let resolved_root = root.canonicalize().map_err(|error| { + miette::miette!( + "trusted supervisor helper runtime {} is unavailable: {error}", + root.display() + ) + })?; + // Kubernetes and Podman preserve root ownership from the supervisor image. + // Docker may materialize the same image-owned runtime in a gateway-user + // cache before bind-mounting it read-only. In that case the immutable + // mount, not its namespace-visible UID, establishes provenance. + let runtime_is_read_only = nix::sys::statvfs::statvfs(&resolved_root) + .is_ok_and(|stat| stat.flags().contains(nix::sys::statvfs::FsFlags::ST_RDONLY)); + let executable = paths .iter() - .copied() - .find(|path| { - let path = Path::new(path); - path.is_absolute() && path.is_file() + .map(|path| resolved_root.join(path)) + .find_map(|path| { + let resolved = path.canonicalize().ok()?; + if !resolved.starts_with(&resolved_root) { + return None; + } + let Ok(metadata) = resolved.metadata() else { + return None; + }; + (metadata.is_file() + && (metadata.uid() == trusted_uid || runtime_is_read_only) + && metadata.mode() & 0o111 != 0 + && metadata.mode() & 0o022 == 0) + .then_some(resolved) }) .ok_or_else(|| { miette::miette!( - "trusted {name} helper not found; checked {}", + "trusted {name} helper not found below {}; checked {}", + resolved_root.display(), paths.join(", ") ) + })?; + let loader = runtime_library_directories(&resolved_root) + .into_iter() + .filter_map(|directory| std::fs::read_dir(directory).ok()) + .flatten() + .filter_map(std::result::Result::ok) + .map(|entry| entry.path()) + .find(|path| is_runtime_loader(path)) + .ok_or_else(|| { + miette::miette!( + "trusted dynamic loader not found below {}", + resolved_root.display() + ) + })? + .canonicalize() + .into_diagnostic()?; + if !loader.starts_with(&resolved_root) { + return Err(miette::miette!("trusted runtime loader escapes its root")); + } + let loader_metadata = loader.metadata().into_diagnostic()?; + if !loader_metadata.is_file() + || (loader_metadata.uid() != trusted_uid && !runtime_is_read_only) + || loader_metadata.mode() & 0o111 == 0 + || loader_metadata.mode() & 0o022 != 0 + { + return Err(miette::miette!( + "trusted runtime loader has unsafe ownership or mode" + )); + } + let library_path = runtime_library_directories(&resolved_root) + .into_iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(":"); + let xtables_path = runtime_library_directories(&resolved_root) + .into_iter() + .map(|directory| directory.join("xtables")) + .find(|path| path.is_dir()) + .unwrap_or_else(|| resolved_root.join("usr/lib/xtables")); + Ok(TrustedHelper { + executable, + loader: Some(loader), + library_path, + xtables_path, + }) +} + +fn runtime_library_directories(root: &Path) -> Vec { + let mut directories = Vec::new(); + for base in ["lib", "lib64", "usr/lib", "usr/lib64"] { + let base = root.join(base); + if !base.is_dir() { + continue; + } + directories.push(base.clone()); + if let Ok(entries) = std::fs::read_dir(base) { + directories.extend( + entries + .filter_map(std::result::Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()), + ); + } + } + directories +} + +fn is_runtime_loader(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + (name.starts_with("ld-musl-") && name.ends_with(".so.1")) + || name == "ld-linux-x86-64.so.2" + || name == "ld-linux-aarch64.so.1" }) } /// Find the nft binary path, checking well-known locations. -fn find_nft() -> Option { - find_trusted_binary("nft", NFT_SEARCH_PATHS) - .ok() - .map(String::from) +fn find_nft(source: HelperSource) -> Option { + find_binary(source, "nft", NFT_SEARCH_PATHS).ok() } -fn find_iptables_legacy() -> Option { - find_trusted_binary("iptables-legacy", IPTABLES_LEGACY_SEARCH_PATHS) - .ok() - .map(String::from) +fn find_iptables_legacy(source: HelperSource) -> Option { + find_binary(source, "iptables-legacy", IPTABLES_LEGACY_SEARCH_PATHS).ok() } -fn find_ip6tables_legacy() -> Option { - find_trusted_binary("ip6tables-legacy", IP6TABLES_LEGACY_SEARCH_PATHS) - .ok() - .map(String::from) +fn find_ip6tables_legacy(source: HelperSource) -> Option { + find_binary(source, "ip6tables-legacy", IP6TABLES_LEGACY_SEARCH_PATHS).ok() } #[cfg(test)] mod tests { use super::*; use std::fs; + use std::os::unix::fs::PermissionsExt as _; // These tests require root and network namespace support // Run with: sudo cargo test -- --ignored #[test] - fn find_trusted_binary_uses_absolute_existing_file() { + fn find_trusted_binary_uses_only_the_supplied_runtime() { let tempdir = tempfile::tempdir().unwrap(); - let helper = tempdir.path().join("ip"); + let helper = tempdir.path().join("usr/sbin/ip"); + fs::create_dir_all(helper.parent().unwrap()).unwrap(); fs::write(&helper, b"test helper").unwrap(); - let helper = helper.to_str().unwrap(); - - assert_eq!( - find_trusted_binary("ip", &["relative-ip", "/missing/ip", helper]).unwrap(), - helper - ); + fs::set_permissions(&helper, fs::Permissions::from_mode(0o755)).unwrap(); + let lib = tempdir.path().join("lib"); + fs::create_dir(&lib).unwrap(); + let loader = lib.join("ld-musl-test.so.1"); + fs::write(&loader, b"test loader").unwrap(); + fs::set_permissions(loader, fs::Permissions::from_mode(0o755)).unwrap(); + + let resolved = find_trusted_binary_in(tempdir.path(), "ip", &["usr/sbin/ip"]).unwrap(); + assert_eq!(resolved.executable, helper); } #[test] fn find_trusted_binary_rejects_missing_helpers() { - let err = - find_trusted_binary("nsenter", &["relative-nsenter", "/missing/nsenter"]).unwrap_err(); + let tempdir = tempfile::tempdir().unwrap(); + let err = find_trusted_binary_in(tempdir.path(), "ip", &["usr/sbin/ip"]).unwrap_err(); - assert!(err.to_string().contains("trusted nsenter helper not found")); + assert!(err.to_string().contains("trusted ip helper not found")); + } + + #[test] + fn trusted_runtime_rejects_helper_symlink_into_workload_root() { + use std::os::unix::fs::symlink; + + let runtime = tempfile::tempdir().unwrap(); + let workload = tempfile::tempdir().unwrap(); + let malicious = workload.path().join("ip"); + fs::write(&malicious, b"malicious workload helper").unwrap(); + fs::set_permissions(&malicious, fs::Permissions::from_mode(0o755)).unwrap(); + let helper = runtime.path().join("usr/sbin/ip"); + fs::create_dir_all(helper.parent().unwrap()).unwrap(); + symlink(&malicious, &helper).unwrap(); + + let error = find_trusted_binary_in(runtime.path(), "ip", &["usr/sbin/ip"]) + .expect_err("helper escaping the trusted runtime must be rejected"); + assert!(error.to_string().contains("trusted ip helper not found")); } #[test] - fn nft_search_paths_are_absolute() { + fn nft_search_paths_are_runtime_relative() { for path in NFT_SEARCH_PATHS { assert!( - path.starts_with('/'), - "NFT_SEARCH_PATHS entry must be absolute: {path}" + !path.starts_with('/'), + "NFT_SEARCH_PATHS entry must be runtime-relative: {path}" ); } } #[test] - fn iptables_legacy_search_paths_are_absolute() { + fn iptables_legacy_search_paths_are_runtime_relative() { for path in IPTABLES_LEGACY_SEARCH_PATHS { assert!( - path.starts_with('/'), - "IPTABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" + !path.starts_with('/'), + "IPTABLES_LEGACY_SEARCH_PATHS entry must be runtime-relative: {path}" ); } } #[test] - fn ip6tables_legacy_search_paths_are_absolute() { + fn ip6tables_legacy_search_paths_are_runtime_relative() { for path in IP6TABLES_LEGACY_SEARCH_PATHS { assert!( - path.starts_with('/'), - "IP6TABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" + !path.starts_with('/'), + "IP6TABLES_LEGACY_SEARCH_PATHS entry must be runtime-relative: {path}" ); } } @@ -1186,25 +1784,112 @@ fe800000000000000000000000000001 02 40 20 80 eth0 } #[test] - fn route_overlap_detects_reserved_pool_collision() { - let reserved = [ - "198.18.1.0/25".parse().unwrap(), - "fd23:6f70:656e:1::/120".parse().unwrap(), - ]; - let routes = "default via 10.200.0.1 dev veth\n198.18.0.0/15 dev eth1\n"; - let (route, pool) = first_route_overlap(routes, &reserved).expect("collision"); - assert_eq!(route.to_string(), "198.18.0.0/15"); - assert_eq!(pool.to_string(), "198.18.1.0/25"); + fn egress_ceiling_verification_accepts_required_live_rules() { + let ruleset = br#"{ + "nftables": [ + {"chain":{"family":"inet","table":"openshell_bypass","name":"output","type":"filter","hook":"output","prio":0,"policy":"drop"}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {"match":{"op":"==","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":"10.200.0.1"}}, + {"match":{"op":"==","left":{"payload":{"protocol":"tcp","field":"dport"}},"right":3128}}, + {"accept":null} + ]}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {"match":{"op":"==","left":{"meta":{"key":"oifname"}},"right":"lo"}}, + {"accept":null} + ]}} + ] + }"#; + + verify_egress_ceiling_json(ruleset, "10.200.0.1", 3128).unwrap(); } #[test] - fn route_overlap_ignores_default_and_unrelated_routes() { - let reserved = [ - "198.18.1.0/25".parse().unwrap(), - "fd23:6f70:656e:1::/120".parse().unwrap(), - ]; - let routes = "default via 10.200.0.1 dev veth\n10.200.0.0/24 dev veth\n"; - assert_eq!(first_route_overlap(routes, &reserved), None); + fn egress_ceiling_verification_rejects_fail_open_chain() { + let ruleset = br#"{ + "nftables": [ + {"chain":{"family":"inet","table":"openshell_bypass","name":"output","hook":"output","policy":"accept"}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[{"match":{"right":"10.200.0.1"}},{"match":{"right":3128}},{"accept":null}]}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[{"match":{"right":"lo"}},{"accept":null}]}} + ] + }"#; + + assert!(verify_egress_ceiling_json(ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_missing_required_allow() { + let ruleset = br#"{ + "nftables": [ + {"chain":{"family":"inet","table":"openshell_bypass","name":"output","hook":"output","policy":"drop"}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[{"match":{"right":"lo"}},{"accept":null}]}} + ] + }"#; + + assert!(verify_egress_ceiling_json(ruleset, "10.200.0.1", 3128).is_err()); + } + + fn ruleset_with_accept_expressions(expressions: &str) -> Vec { + format!( + r#"{{"nftables":[ + {{"chain":{{"family":"inet","table":"openshell_bypass","name":"output","type":"filter","hook":"output","prio":0,"policy":"drop"}}}}, + {{"rule":{{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {{"match":{{"op":"==","left":{{"payload":{{"protocol":"ip","field":"daddr"}}}},"right":"10.200.0.1"}}}}, + {{"match":{{"op":"==","left":{{"payload":{{"protocol":"tcp","field":"dport"}}}},"right":3128}}}},{{"accept":null}}]}}}}, + {{"rule":{{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {{"match":{{"op":"==","left":{{"meta":{{"key":"oifname"}}}},"right":"lo"}}}},{{"accept":null}}]}}}}, + {{"rule":{{"family":"inet","table":"openshell_bypass","chain":"output","expr":{expressions}}}}} + ]}}"# + ) + .into_bytes() + } + + #[test] + fn egress_ceiling_verification_rejects_unconditional_accept() { + let ruleset = ruleset_with_accept_expressions(r#"[{"accept":null}]"#); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_unrelated_matching_metadata() { + let ruleset = ruleset_with_accept_expressions( + r#"[{"comment":{"address":"10.200.0.1","port":3128}},{"accept":null}]"#, + ); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_wrong_protocol_or_operator() { + for expressions in [ + r#"[{"match":{"op":"==","left":{"payload":{"protocol":"udp","field":"dport"}},"right":3128}},{"accept":null}]"#, + r#"[{"match":{"op":"!=","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":"10.200.0.1"}},{"accept":null}]"#, + ] { + let ruleset = ruleset_with_accept_expressions(expressions); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + } + + #[test] + fn egress_ceiling_verification_rejects_extra_destination_allow() { + let ruleset = ruleset_with_accept_expressions( + r#"[{"match":{"op":"==","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":"203.0.113.1"}},{"accept":null}]"#, + ); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_jump_to_unverified_chain() { + let ruleset = ruleset_with_accept_expressions(r#"[{"jump":{"target":"unverified"}}]"#); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_ignores_accept_in_another_chain() { + let mut document: serde_json::Value = + serde_json::from_slice(&ruleset_with_accept_expressions(r#"[{"accept":null}]"#)) + .unwrap(); + document["nftables"][3]["rule"]["chain"] = serde_json::json!("other"); + let ruleset = serde_json::to_vec(&document).unwrap(); + verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).unwrap(); } #[test] @@ -1214,8 +1899,8 @@ fe800000000000000000000000000001 02 40 20 80 eth0 let name = ns.name().to_string(); // Verify namespace exists - let ns_path = openshell_core::container_paths::netns_path(&name); - assert!(ns_path.exists(), "Namespace file should exist"); + let ns_path = format!("/var/run/netns/{name}"); + assert!(Path::new(&ns_path).exists(), "Namespace file should exist"); // Verify IPs are set correctly assert_eq!( @@ -1236,4 +1921,181 @@ fe800000000000000000000000000001 02 40 20 80 eth0 "Namespace should be cleaned up" ); } + + #[test] + #[ignore = "requires root privileges"] + fn installed_egress_ceiling_round_trips_through_kernel() { + let ns = NetworkNamespace::create_conformant().expect("create conformant namespace"); + ns.install_egress_ceiling(3128).expect("install ceiling"); + ns.verify_egress_ceiling(3128).expect("verify ceiling"); + } + + #[test] + #[ignore = "requires root privileges"] + fn installed_egress_ceiling_allows_only_proxy_tcp() { + use std::time::Duration; + + #[allow(unsafe_code)] + fn enter_namespace(ns_fd: RawFd) { + // SAFETY: the owning NetworkNamespace remains alive until every + // test thread has joined, so the descriptor stays valid. + let result = unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) }; + assert_eq!(result, 0, "enter workload network namespace"); + } + + let ns = NetworkNamespace::create_conformant().expect("create conformant namespace"); + let ns_fd = ns.ns_fd().expect("network namespace fd"); + let host_ip = ns.host_ip(); + + let alternate_host_ip: std::net::Ipv4Addr = "10.200.0.3".parse().unwrap(); + run_ip( + ns.helper_source, + &["addr", "add", "10.200.0.3/24", "dev", &ns.veth_host], + ) + .expect("add alternate routed IPv4 destination"); + let host_ipv6: std::net::Ipv6Addr = "fd00:200::1".parse().unwrap(); + run_ip( + ns.helper_source, + &[ + "-6", + "addr", + "add", + "fd00:200::1/64", + "dev", + &ns.veth_host, + "nodad", + ], + ) + .expect("add host IPv6 destination"); + run_ip_netns( + ns.helper_source, + ns.name(), + &[ + "-6", + "addr", + "add", + "fd00:200::2/64", + "dev", + &ns.veth_sandbox, + "nodad", + ], + ) + .expect("add workload IPv6 source"); + + // Positive controls prove each route/protocol works before the ceiling + // is installed, so later denial cannot pass because of broken setup. + let ipv4_control = + std::net::TcpListener::bind((alternate_host_ip, 0)).expect("bind IPv4 control"); + let ipv4_control_address = ipv4_control.local_addr().unwrap(); + assert!( + std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&ipv4_control_address, Duration::from_secs(1)) + }) + .join() + .expect("IPv4 control thread") + .is_ok(), + "alternate IPv4 route must work before enforcement" + ); + + let ipv6_control = std::net::TcpListener::bind((host_ipv6, 0)).expect("bind IPv6 control"); + let ipv6_control_address = ipv6_control.local_addr().unwrap(); + assert!( + std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&ipv6_control_address, Duration::from_secs(1)) + }) + .join() + .expect("IPv6 control thread") + .is_ok(), + "IPv6 route must work before enforcement" + ); + + let udp_control = + std::net::UdpSocket::bind((host_ip, 0)).expect("bind UDP positive control"); + udp_control + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let udp_control_address = udp_control.local_addr().unwrap(); + std::thread::spawn(move || { + enter_namespace(ns_fd); + let socket = std::net::UdpSocket::bind("0.0.0.0:0").expect("bind control UDP"); + socket.send_to(b"control", udp_control_address) + }) + .join() + .expect("UDP control thread") + .expect("send UDP positive control"); + let mut control = [0_u8; 7]; + udp_control + .recv_from(&mut control) + .expect("UDP route must work before enforcement"); + assert_eq!(&control, b"control"); + + let proxy = std::net::TcpListener::bind((host_ip, 0)).expect("bind proxy listener"); + let proxy_address = proxy.local_addr().expect("proxy address"); + ns.install_egress_ceiling(proxy_address.port()) + .expect("install ceiling"); + + let allowed = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&proxy_address, Duration::from_secs(1)) + }); + proxy + .set_nonblocking(true) + .expect("set proxy listener nonblocking"); + assert!(allowed.join().expect("allowed-connect thread").is_ok()); + + let direct = std::net::TcpListener::bind((host_ip, 0)).expect("bind direct listener"); + let direct_address = direct.local_addr().expect("direct address"); + let denied = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&direct_address, Duration::from_millis(300)) + }); + assert!( + denied.join().expect("denied-connect thread").is_err(), + "direct TCP must not bypass mediation" + ); + + let alternate = std::net::TcpListener::bind((alternate_host_ip, proxy_address.port())) + .expect("bind alternate routed listener"); + let alternate_address = alternate.local_addr().unwrap(); + let denied = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&alternate_address, Duration::from_millis(300)) + }); + assert!( + denied.join().expect("alternate-connect thread").is_err(), + "the proxy port at another routed IPv4 destination must be denied" + ); + + let ipv6 = std::net::TcpListener::bind((host_ipv6, proxy_address.port())) + .expect("bind IPv6 observer"); + let ipv6_address = ipv6.local_addr().unwrap(); + let denied = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&ipv6_address, Duration::from_millis(300)) + }); + assert!( + denied.join().expect("IPv6-connect thread").is_err(), + "direct IPv6 TCP must not bypass mediation" + ); + + let udp = std::net::UdpSocket::bind((host_ip, proxy_address.port())) + .expect("bind UDP observer at proxy destination"); + udp.set_read_timeout(Some(Duration::from_millis(300))) + .expect("set UDP timeout"); + let udp_address = udp.local_addr().expect("UDP address"); + let udp_send = std::thread::spawn(move || { + enter_namespace(ns_fd); + let socket = std::net::UdpSocket::bind("0.0.0.0:0").expect("bind workload UDP"); + socket.send_to(b"bypass", udp_address) + }) + .join() + .expect("UDP thread"); + let mut byte = [0_u8; 1]; + assert!( + udp_send.is_err() || udp.recv_from(&mut byte).is_err(), + "direct UDP must not bypass mediation" + ); + } } diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index cb1399d830..3da9c6ad18 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -29,8 +29,10 @@ use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::path::Path; use std::path::PathBuf; use std::process::Stdio; +use std::sync::Arc; #[cfg(target_os = "linux")] use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, Ordering}; #[cfg(target_os = "linux")] use std::sync::mpsc; use tokio::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; @@ -656,8 +658,8 @@ pub struct ProcessHandle { child: Child, pid: u32, io: Option, - #[cfg(target_os = "linux")] - managed_child: Option, + terminal: Arc, + signal_lock: Arc>, } /// Supervisor-owned canonical-process I/O. These handles outlive individual @@ -930,8 +932,8 @@ impl ProcessHandle { child, pid, io: Some(io), - #[cfg(target_os = "linux")] - managed_child, + terminal: Arc::new(AtomicBool::new(false)), + signal_lock: Arc::new(std::sync::Mutex::new(())), }) } @@ -1078,6 +1080,8 @@ impl ProcessHandle { child, pid, io: Some(io), + terminal: Arc::new(AtomicBool::new(false)), + signal_lock: Arc::new(std::sync::Mutex::new(())), }) } @@ -1092,6 +1096,12 @@ impl ProcessHandle { self.io.take().expect("canonical process I/O already taken") } + /// Shared state used by an independent boundary signal handle. + #[must_use] + pub fn signaling_state(&self) -> (Arc, Arc>) { + (self.terminal.clone(), self.signal_lock.clone()) + } + /// Wait for the process to exit. /// /// # Errors @@ -1099,6 +1109,11 @@ impl ProcessHandle { /// Returns an error if waiting fails. pub async fn wait(&mut self) -> std::io::Result { let status = self.child.wait().await; + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.terminal.store(true, Ordering::Release); #[cfg(target_os = "linux")] if let Some(child) = self.managed_child.take() { managed_children::unregister(child); @@ -1111,6 +1126,11 @@ impl ProcessHandle { pub fn try_wait(&mut self) -> std::io::Result> { let status = self.child.try_wait()?; if status.is_some() { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.terminal.store(true, Ordering::Release); #[cfg(target_os = "linux")] if let Some(child) = self.managed_child.take() { managed_children::unregister(child); @@ -1125,6 +1145,13 @@ impl ProcessHandle { /// /// Returns an error if the signal cannot be sent. pub fn signal(&self, sig: Signal) -> Result<()> { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("process has exited")); + } let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); signal::kill(Pid::from_raw(pid), sig).into_diagnostic() } diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index 8c47e789ba..a5f6bf58b2 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -26,6 +26,7 @@ use crate::netns::NetworkNamespace; use openshell_core::policy::{NetworkMode, SandboxPolicy}; use openshell_core::proposals::AgentProposals; use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward}; #[cfg(target_os = "linux")] use openshell_core::activity::ActivitySender; @@ -237,11 +238,6 @@ pub async fn run_process( // CLONE_NEWNET) so it lands inside the workload's network namespace. // Without this, SSH-spawned shells run in the host namespace and bypass // the proxy entirely. - #[cfg(target_os = "linux")] - let ssh_netns_fd = netns.and_then(NetworkNamespace::ns_fd); - #[cfg(not(target_os = "linux"))] - let ssh_netns_fd: Option = None; - #[cfg(target_os = "linux")] let mut handle = ProcessHandle::spawn( program, @@ -282,20 +278,45 @@ pub async fn run_process( #[cfg(not(target_os = "linux"))] let ssh_proxy_url = ssh_proxy_url_for_policy(policy, None); + #[cfg(target_os = "linux")] + let boundary_netns_fd = netns + .map(NetworkNamespace::try_clone_ns_fd) + .transpose()? + .flatten() + .map(Arc::new); + #[cfg(not(target_os = "linux"))] + let boundary_netns_fd = None; + let boundary_runtime = crate::boundary_io::BoundaryRuntimeState::new(); + let user_environment: std::collections::HashMap = + std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) + .ok() + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_default(); + let port_forward: Arc = + Arc::new(crate::boundary_io::NetnsPortForward::new( + boundary_netns_fd.clone(), + Some(boundary_runtime.clone()), + )); + let boundary_exec: Arc = + Arc::new(crate::boundary_exec::LocalBoundaryExec::new( + policy.clone(), + workspace.owned_root(), + boundary_netns_fd, + ssh_proxy_url.clone(), + ca_file_paths.clone().map(Arc::new), + provider_credentials.clone(), + user_environment, + resolved_process_identity, + enforcement_mode, + boundary_runtime.clone(), + )); + let ssh_socket_path: Option = ssh_socket_path.map(std::path::PathBuf::from); if let Some(listen_path) = ssh_socket_path.clone() { - let policy_clone = policy.clone(); - let workspace_clone = workspace.clone(); - let proxy_url = ssh_proxy_url; - let netns_fd = ssh_netns_fd; let ca_paths = ca_file_paths.clone(); - let provider_credentials_clone = provider_credentials.clone(); - let main_session_clone = Arc::clone(&main_session); - let user_env_clone: std::collections::HashMap = - std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) - .ok() - .and_then(|json| serde_json::from_str(&json).ok()) - .unwrap_or_default(); + let ssh_port_forward = port_forward.clone(); + let ssh_boundary_exec = boundary_exec.clone(); + let ssh_main_session = Arc::clone(&main_session); let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); @@ -304,17 +325,12 @@ pub async fn run_process( if let Err(err) = crate::ssh::run_ssh_server( listen_path, ssh_ready_tx, - policy_clone, - workspace_clone, - netns_fd, - proxy_url, ca_paths, - provider_credentials_clone, - user_env_clone, - resolved_process_identity, enforcement_mode, shared_ssh_socket, - main_session_clone, + ssh_port_forward, + ssh_boundary_exec, + Some(ssh_main_session), ) .await { @@ -374,7 +390,7 @@ pub async fn run_process( endpoint.to_string(), id.to_string(), socket.clone(), - ssh_netns_fd, + port_forward, None, Arc::clone(&supervisor_terminating), main_instance_id.clone(), @@ -484,6 +500,7 @@ pub async fn run_process( } supervisor_terminating.store(true, Ordering::Release); + boundary_runtime.deactivate(); if let Some(task) = supervisor_session_task { task.abort(); } @@ -718,7 +735,7 @@ fn ssh_proxy_url_for_policy( /// /// Best-effort: any failure (no gateway, RPC error, install failure) is /// logged but does not fail sandbox startup. -async fn install_initial_agent_skill( +pub(crate) async fn install_initial_agent_skill( sandbox_id: Option<&str>, openshell_endpoint: Option<&str>, agent_proposals: &AgentProposals, diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 6d45e31777..0b4101160f 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -8,19 +8,16 @@ use crate::main_session::{MainOutput, MainSession}; #[cfg(target_os = "linux")] use crate::managed_children; use crate::process::{ - ProcessEnforcementMode, ResolvedProcessIdentity, ResolvedWorkspace, - drop_privileges_with_identity, is_supervisor_only_env_var, session_user_and_home, + ProcessEnforcementMode, ResolvedProcessIdentity, drop_privileges_with_identity, + is_supervisor_only_env_var, }; use crate::sandbox; -#[cfg(unix)] -use libc; use miette::{IntoDiagnostic, Result}; use nix::pty::{Winsize, openpty}; use nix::unistd::setsid; use openshell_core::VERSION; -use openshell_core::net::set_tcp_nodelay_best_effort; +use openshell_core::net::{connect_tcp_nodelay_best_effort, set_tcp_nodelay_best_effort}; use openshell_core::policy::SandboxPolicy; -use openshell_core::provider_credentials::ProviderCredentialState; use openshell_ocsf::{ ActionId, ActivityId, DispositionId, SeverityId, SshActivityBuilder, StatusId, ocsf_emit, }; @@ -30,7 +27,7 @@ use russh::{ChannelId, ChannelOpenFailure, Sig}; use std::borrow::Cow; use std::collections::HashMap; use std::io::{Read, Write}; -use std::os::fd::{AsRawFd, RawFd}; +use std::os::fd::{AsRawFd, OwnedFd, RawFd}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::{Arc, mpsc}; @@ -39,6 +36,28 @@ use tokio::net::UnixListener; use tracing::warn; const NO_LOGIN_SHELL_ENV: (&str, &str) = ("OPENSHELL_NO_LOGIN_SHELL", "1"); +const MAIN_DETACH_PREFIX: u8 = 0x10; +const MAIN_DETACH_KEY: u8 = 0x11; + +fn filter_main_detach_sequence(prefix_pending: &mut bool, data: &[u8]) -> (Vec, bool) { + let mut forward = Vec::with_capacity(data.len() + usize::from(*prefix_pending)); + for &byte in data { + if *prefix_pending { + if byte == MAIN_DETACH_KEY { + *prefix_pending = false; + return (forward, true); + } + forward.push(MAIN_DETACH_PREFIX); + *prefix_pending = false; + } + if byte == MAIN_DETACH_PREFIX { + *prefix_pending = true; + } else { + forward.push(byte); + } + } + (forward, false) +} /// Perform SSH server initialization: generate a host key, build the config, /// and bind the Unix socket listener. Extracted so that startup errors can be @@ -58,7 +77,6 @@ fn ssh_server_init( let mut rng = rand::rng(); let host_key = PrivateKey::random(&mut rng, Algorithm::Ed25519).into_diagnostic()?; - // TODO: while building the SSH config, refactor the server_id to be "SSH-2.0-OpenShell_" from `openshell_core::VERSION` let mut config = russh::server::Config { server_id: russh::SshId::Standard(Cow::Owned(format!("SSH-2.0-OpenShell_{VERSION}"))), auth_rejection_time: Duration::from_secs(1), @@ -119,19 +137,14 @@ fn ssh_server_init( pub async fn run_ssh_server( listen_path: PathBuf, ready_tx: tokio::sync::oneshot::Sender>, - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, ca_file_paths: Option<(PathBuf, PathBuf)>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, shared_socket: bool, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Result<()> { - let (listener, config, ca_paths) = match ssh_server_init( + let (listener, config, _ca_paths) = match ssh_server_init( &listen_path, &ca_file_paths, enforcement_mode, @@ -151,186 +164,27 @@ pub async fn run_ssh_server( } }; - let mut consecutive_resource_errors: u32 = 0; - let mut consecutive_unknown_errors: u32 = 0; - loop { - match listener.accept().await { - Ok((stream, _peer)) => { - consecutive_resource_errors = 0; - consecutive_unknown_errors = 0; - let config = config.clone(); - let policy = policy.clone(); - let workspace = workspace.clone(); - let proxy_url = proxy_url.clone(); - let ca_paths = ca_paths.clone(); - let provider_credentials = provider_credentials.clone(); - let user_environment = user_environment.clone(); - let main_session = Arc::clone(&main_session); - - tokio::spawn(async move { - if let Err(err) = handle_connection( - stream, - config, - policy, - workspace, - netns_fd, - proxy_url, - ca_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, - ) - .await - { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .message(format!("SSH connection failed: {err}")) - .build() - ); - } - }); - } - Err(err) => { - match classify_ssh_accept_error( - &err, - &mut consecutive_resource_errors, - &mut consecutive_unknown_errors, - ) { - SshAcceptAction::Terminal => { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message(format!( - "SSH accept loop exiting on terminal error: {err}" - )) - .build() - ); - break; - } - SshAcceptAction::Retry { backoff, severity } => { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(severity) - .status(StatusId::Failure) - .message(format!( - "SSH accept error (retrying in {}ms): {err}", - backoff.as_millis(), - )) - .build() - ); - tokio::time::sleep(backoff).await; - } - } - } - } - } - - Ok(()) -} - -const MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS: u32 = 10; - -#[derive(Debug, PartialEq)] -enum SshAcceptAction { - Terminal, - Retry { - backoff: Duration, - severity: SeverityId, - }, -} - -fn classify_ssh_accept_error( - err: &std::io::Error, - consecutive_resource_errors: &mut u32, - consecutive_unknown_errors: &mut u32, -) -> SshAcceptAction { - #[cfg(unix)] - if matches!( - err.raw_os_error(), - Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK) - ) { - return SshAcceptAction::Terminal; - } + let (stream, _peer) = listener.accept().await.into_diagnostic()?; + let config = config.clone(); + let port_forward = port_forward.clone(); + let boundary_exec = boundary_exec.clone(); + let main_session = main_session.clone(); - #[cfg(unix)] - if matches!( - err.raw_os_error(), - Some( - libc::EMFILE - | libc::ENFILE - | libc::ENOBUFS - | libc::ENOMEM - | libc::ECONNABORTED - | libc::ECONNRESET - | libc::EINTR - | libc::ENETDOWN - | libc::EPROTO - | libc::ENOPROTOOPT - | libc::EHOSTDOWN - | libc::EHOSTUNREACH - | libc::EOPNOTSUPP - | libc::ENETUNREACH - | libc::ENOSR - | libc::ESOCKTNOSUPPORT - | libc::EPROTONOSUPPORT - | libc::ETIMEDOUT - ) - ) { - *consecutive_unknown_errors = 0; - - #[cfg(unix)] - let is_resource_pressure = matches!( - err.raw_os_error(), - Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM | libc::ENOSR) - ); - #[cfg(not(unix))] - let is_resource_pressure = false; - - if is_resource_pressure { - *consecutive_resource_errors = consecutive_resource_errors.saturating_add(1); - let backoff_ms = 100u64 - .saturating_mul(1u64 << (*consecutive_resource_errors).min(7).saturating_sub(1)) - .min(5_000); - return SshAcceptAction::Retry { - backoff: Duration::from_millis(backoff_ms), - severity: SeverityId::Medium, - }; - } - - *consecutive_resource_errors = 0; - return SshAcceptAction::Retry { - backoff: Duration::from_millis(100), - severity: SeverityId::Low, - }; - } - - #[cfg(unix)] - #[cfg(target_os = "linux")] - if matches!(err.raw_os_error(), Some(libc::ENONET)) { - *consecutive_unknown_errors = 0; - *consecutive_resource_errors = 0; - return SshAcceptAction::Retry { - backoff: Duration::from_millis(100), - severity: SeverityId::Low, - }; - } - - *consecutive_unknown_errors = consecutive_unknown_errors.saturating_add(1); - if *consecutive_unknown_errors >= MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS { - return SshAcceptAction::Terminal; - } - SshAcceptAction::Retry { - backoff: Duration::from_millis(100), - severity: SeverityId::Low, + tokio::spawn(async move { + if let Err(err) = + handle_connection(stream, config, port_forward, boundary_exec, main_session).await + { + ocsf_emit!( + SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .message(format!("SSH connection failed: {err}")) + .build() + ); + } + }); } } @@ -338,16 +192,9 @@ fn classify_ssh_accept_error( async fn handle_connection( stream: tokio::net::UnixStream, config: Arc, - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Result<()> { // Access is gated by the Unix-socket filesystem permissions (root-only), // not by an application-level preface. The supervisor bridges the @@ -363,18 +210,7 @@ async fn handle_connection( .build() ); - let handler = SshHandler::new( - policy, - workspace, - netns_fd, - proxy_url, - ca_file_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, - ); + let handler = SshHandler::new(port_forward, boundary_exec, main_session); russh::server::run_stream(config, stream, handler) .await .map_err(|err| miette::miette!("ssh stream error: {err}"))?; @@ -387,13 +223,12 @@ async fn handle_connection( /// sender. This allows `window_change_request` to resize the correct PTY when /// multiple channels are open simultaneously (e.g. parallel shells, shell + /// sftp, etc.). -// Several independent per-channel boolean flags (login-shell opt-out and the -// main-attachment state bits) legitimately live side by side here. #[allow(clippy::struct_excessive_bools)] #[derive(Default)] struct ChannelState { input_sender: Option, - pty_master: Option, + process: Option>, + terminal: Option>, pty_request: Option, no_login_shell: bool, main_input_owner: Option, @@ -403,37 +238,6 @@ struct ChannelState { main_output_task: Option, } -const MAIN_DETACH_PREFIX: u8 = 0x10; // Ctrl-P -const MAIN_DETACH_KEY: u8 = 0x11; // Ctrl-Q - -/// Remove the `OpenShell` detach sequence from canonical-main input. -/// -/// A trailing Ctrl-P remains pending across SSH data frames. If the following -/// byte is not Ctrl-Q, both bytes are forwarded unchanged. Bytes after a -/// completed detach sequence are discarded because the attachment is closing. -fn filter_main_detach_sequence(prefix_pending: &mut bool, data: &[u8]) -> (Vec, bool) { - let mut forward = Vec::with_capacity(data.len() + usize::from(*prefix_pending)); - - for &byte in data { - if *prefix_pending { - if byte == MAIN_DETACH_KEY { - *prefix_pending = false; - return (forward, true); - } - forward.push(MAIN_DETACH_PREFIX); - *prefix_pending = false; - } - - if byte == MAIN_DETACH_PREFIX { - *prefix_pending = true; - } else { - forward.push(byte); - } - } - - (forward, false) -} - enum InputSender { Process(mpsc::Sender>), Main(tokio::sync::mpsc::Sender>), @@ -454,28 +258,26 @@ impl InputSender { } struct SshHandler { - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + /// Loopback port-forward, injected by the orchestrator (RFC 0012). In-pod + /// this connects from inside the workload netns; a delegated backend + /// tunnels into its guest. The handler does not know which. + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, channels: HashMap, } impl Drop for SshHandler { fn drop(&mut self) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; for state in self.channels.values_mut() { if state.main_attached { - self.main_session.end_terminal_attachment(); - state.main_attached = false; + main_session.end_terminal_attachment(); } if let Some(owner) = state.main_input_owner.take() { - self.main_session.release_input(owner); + main_session.release_input(owner); } if let Some(task) = state.main_output_task.take() { task.abort(); @@ -485,29 +287,14 @@ impl Drop for SshHandler { } impl SshHandler { - #[allow(clippy::too_many_arguments)] fn new( - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Self { Self { - policy, - workspace, - netns_fd, - proxy_url, - ca_file_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, + port_forward, + boundary_exec, main_session, channels: HashMap::new(), } @@ -550,15 +337,23 @@ impl russh::server::Handler for SshHandler { channel: ChannelId, _session: &mut Session, ) -> Result<(), Self::Error> { - if let Some(state) = self.channels.remove(&channel) { - if state.main_attached { - self.main_session.end_terminal_attachment(); - } - if let Some(owner) = state.main_input_owner { - self.main_session.release_input(owner); + if let Some(mut state) = self.channels.remove(&channel) { + if state.main_attached + && let Some(main_session) = self.main_session.as_ref() + { + main_session.end_terminal_attachment(); + if let Some(owner) = state.main_input_owner.take() { + main_session.release_input(owner); + } + if let Some(task) = state.main_output_task.take() { + task.abort(); + } + return Ok(()); } - if let Some(task) = state.main_output_task { - task.abort(); + if let Some(process) = state.process { + // Channel ownership defines the exec lifetime. Closing an SSH + // channel must not strand an in-boundary process. + let _ = process.terminate().await; } } Ok(()) @@ -574,12 +369,6 @@ impl russh::server::Handler for SshHandler { reply: ChannelOpenHandle, _session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - reply - .reject(ChannelOpenFailure::AdministrativelyProhibited) - .await; - return Ok(()); - } // Validate port range before truncating u32 -> u16. The SSH protocol // uses u32 for ports, but valid TCP ports are 0-65535. Without this // check, port 65537 truncates to port 1 (privileged). @@ -621,15 +410,23 @@ impl russh::server::Handler for SshHandler { // SSH protocol port is bounded by u32 but only u16 is meaningful; // saturate as a guard for malformed clients. let port = u16::try_from(port_to_connect).unwrap_or(u16::MAX); - let netns_fd = self.netns_fd; - // Confirm the channel before spawning: the task below writes to it, and - // the peer must see the open-confirmation first. + // Build the loopback target up front. The host already passed + // `is_loopback_host`, and `LoopbackTarget::new` re-validates the parsed + // address (defense in depth) before the connect. + let Some(target) = loopback_ip(&host).and_then(|ip| { + openshell_isolation_interface::contract::LoopbackTarget::new(ip, port).ok() + }) else { + reply + .reject(ChannelOpenFailure::AdministrativelyProhibited) + .await; + return Ok(()); + }; + let port_forward = self.port_forward.clone(); reply.accept().await; tokio::spawn(async move { - let addr = format!("{host}:{port}"); - let tcp = match connect_in_netns(&addr, netns_fd).await { + let mut tcp_stream = match port_forward.connect(target).await { Ok(stream) => stream, Err(err) => { ocsf_emit!( @@ -637,7 +434,9 @@ impl russh::server::Handler for SshHandler { .activity(ActivityId::Fail) .severity(SeverityId::Low) .status(StatusId::Failure) - .message(format!("direct-tcpip: failed to connect to {addr}: {err}")) + .message(format!( + "direct-tcpip: failed to connect to {host}:{port}: {err}" + )) .build() ); let _ = channel.close().await; @@ -646,7 +445,6 @@ impl russh::server::Handler for SshHandler { }; let mut channel_stream = channel.into_stream(); - let mut tcp_stream = tcp; let _ = tokio::io::copy_bidirectional(&mut channel_stream, &mut tcp_stream).await; }); @@ -694,18 +492,15 @@ impl russh::server::Handler for SshHandler { return Ok(()); }; if state.main_attached { - self.main_session - .resize(col_width, row_height, pixel_width, pixel_height); - } else if let Some(master) = state.pty_master.as_ref() { - let winsize = Winsize { - ws_row: to_u16(row_height.max(1)), - ws_col: to_u16(col_width.max(1)), - ws_xpixel: to_u16(pixel_width), - ws_ypixel: to_u16(pixel_height), - }; - if let Err(e) = unsafe_pty::set_winsize(master.as_raw_fd(), winsize) { - warn!("failed to resize PTY for channel {channel:?}: {e}"); + if let Some(main_session) = self.main_session.as_ref() { + main_session.resize(col_width, row_height, pixel_width, pixel_height); } + } else if let Some(terminal) = state.terminal.as_ref() + && let Err(e) = terminal + .resize(to_u16(col_width.max(1)), to_u16(row_height.max(1))) + .await + { + warn!("failed to resize PTY for channel {channel:?}: {e}"); } Ok(()) } @@ -715,10 +510,6 @@ impl russh::server::Handler for SshHandler { channel: ChannelId, session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - session.channel_failure(channel)?; - return Ok(()); - } session.channel_success(channel)?; // Only allocate a PTY when the client explicitly requested one via // pty_request. VS Code Remote-SSH sends shell_request *without* a @@ -726,7 +517,7 @@ impl russh::server::Handler for SshHandler { // endings. Forcing a PTY here caused CRLF translation which made // VS Code misdetect the platform as Windows (and then try to run // `powershell`). - self.start_shell(channel, session.handle(), None)?; + self.start_shell(channel, session.handle(), None).await?; Ok(()) } @@ -736,16 +527,13 @@ impl russh::server::Handler for SshHandler { data: &[u8], session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - session.channel_failure(channel)?; - return Ok(()); - } session.channel_success(channel)?; let command = String::from_utf8_lossy(data).trim().to_string(); if command.is_empty() { return Ok(()); } - self.start_shell(channel, session.handle(), Some(command))?; + self.start_shell(channel, session.handle(), Some(command)) + .await?; Ok(()) } @@ -756,12 +544,15 @@ impl russh::server::Handler for SshHandler { session: &mut Session, ) -> Result<(), Self::Error> { if name == "openshell-main" { - if !self.channels.contains_key(&channel) { - return Err(anyhow::anyhow!( - "subsystem_request on unknown channel {channel:?}" - )); + let Some(main_session) = self.main_session.clone() else { + session.channel_failure(channel)?; + return Ok(()); + }; + if !self.channels.contains_key(&channel) || main_session.finished() { + session.channel_failure(channel)?; + return Ok(()); } - if self.main_session.begin_terminal_attachment().is_err() { + if main_session.begin_terminal_attachment().is_err() { session.channel_failure(channel)?; return Ok(()); } @@ -771,34 +562,31 @@ impl russh::server::Handler for SshHandler { .expect("main channel existence checked above"); state.main_attached = true; if let Some(pty) = state.pty_request.take() { - self.main_session.resize( + main_session.resize( pty.col_width, pty.row_height, pty.pixel_width, pty.pixel_height, ); } - let (input, input_warning) = if state.main_read_only { + let (input, warning) = if state.main_read_only { (None, None) } else { - match self.main_session.acquire_input() { + match main_session.acquire_input() { Ok((owner, input)) => { state.main_input_owner = Some(owner); (Some(InputSender::Main(input)), None) } - Err(error) => { - warn!(%error, "main process input lease unavailable; attaching read-only"); - (None, Some(error)) - } + Err(error) => (None, Some(error)), } }; - state.main_detach_prefix_pending = false; state.input_sender = input; - let mut output = self.main_session.subscribe(); - let terminal_delivery = Arc::clone(&self.main_session); + state.main_detach_prefix_pending = false; + let mut output = main_session.subscribe(); + let terminal_delivery = main_session.clone(); let handle = session.handle(); session.channel_success(channel)?; - if let Some(error) = input_warning { + if let Some(error) = warning { let _ = handle .extended_data( channel, @@ -810,13 +598,13 @@ impl russh::server::Handler for SshHandler { let output_task = tokio::spawn(async move { loop { match output.recv().await { + Ok(MainOutput::Exit(code)) => { + terminal_delivery.wait_for_terminal_reported().await; + let _ = + send_main_output(&handle, channel, MainOutput::Exit(code)).await; + break; + } Ok(event) => { - if let MainOutput::Exit(code) = event { - terminal_delivery.wait_for_terminal_reported().await; - let _ = send_main_output(&handle, channel, MainOutput::Exit(code)) - .await; - break; - } let _ = send_main_output(&handle, channel, event).await; } Err(error) => { @@ -840,31 +628,24 @@ impl russh::server::Handler for SshHandler { if let Some(state) = self.channels.get_mut(&channel) { state.main_output_task = Some(output_task.abort_handle()); } - } else if name == "sftp" && !self.main_session.finished() { + } else if name == "sftp" { session.channel_success(channel)?; // sftp-server speaks the SFTP binary protocol over stdin/stdout, - // which is exactly what spawn_pipe_exec wires up. This enables + // which the boundary executor preserves as separate pipes. This enables // modern scp (SFTP-based, OpenSSH 9.0+) and SFTP clients to // transfer files into and out of the sandbox. - let input_sender = spawn_pipe_exec( - &self.policy, - &self.workspace, - Some("/usr/lib/openssh/sftp-server".to_string()), - false, - session.handle(), + self.start_exec_spec( channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &self.provider_credentials.child_env_with_gcp_resolved(), - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - let state = self.channels.get_mut(&channel).ok_or_else(|| { - anyhow::anyhow!("subsystem_request on unknown channel {channel:?}") - })?; - state.input_sender = Some(InputSender::Process(input_sender)); + session.handle(), + openshell_isolation_interface::contract::ExecSpec { + program: "/usr/lib/openssh/sftp-server".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }, + ) + .await?; } else { ocsf_emit!( SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -889,11 +670,9 @@ impl russh::server::Handler for SshHandler { ) -> Result<(), Self::Error> { // Accept the env request so the client knows we handled it, but we // don't actually propagate arbitrary variables — the sandbox - // environment is controlled via policy. We must reply so VSCode - // doesn't stall. Two exceptions carry supervisor signals the SSH - // protocol has no native field for: - // - OPENSHELL_NO_LOGIN_SHELL: gateway login-shell opt-out. - // - OPENSHELL_MAIN_READ_ONLY: read-only main attachment. + // environment is controlled via policy. The login-shell opt-out is a + // supervisor signal carried over SSH because the protocol has no + // native field for it. if variable_name == NO_LOGIN_SHELL_ENV.0 && let Some(state) = self.channels.get_mut(&channel) { @@ -919,38 +698,17 @@ impl russh::server::Handler for SshHandler { warn!("data on unknown channel {channel:?}"); return Ok(()); }; - - let main_attached = state.main_attached; - let (forward, detach) = if main_attached { + let (forward, detach) = if state.main_attached { filter_main_detach_sequence(&mut state.main_detach_prefix_pending, data) } else { (data.to_vec(), false) }; - let send_error = (!forward.is_empty()) + let error = (!forward.is_empty()) .then(|| state.input_sender.as_ref()?.send(forward).err()) .flatten(); - - if let Some(error) = send_error { - let handle = session.handle(); - if main_attached { - self.close_main_attachment(channel, handle, Some(error)) - .await; - } else { - let _ = handle - .extended_data( - channel, - 1, - format!("openshell: {error}; closing attachment\n").into_bytes(), - ) - .await; - let _ = handle.close(channel).await; - } - return Ok(()); - } - if detach { - self.close_main_attachment(channel, session.handle(), None) + if state.main_attached && (detach || error.is_some()) { + self.close_main_attachment(channel, session.handle(), error) .await; - return Ok(()); } Ok(()) } @@ -967,8 +725,9 @@ impl russh::server::Handler for SshHandler { if let Some(state) = self.channels.get_mut(&channel) { if state.main_attached && let Some(owner) = state.main_input_owner.take() + && let Some(main_session) = self.main_session.as_ref() { - self.main_session.release_input(owner); + main_session.release_input(owner); } state.input_sender.take(); state.main_detach_prefix_pending = false; @@ -984,47 +743,192 @@ impl russh::server::Handler for SshHandler { signal: Sig, _session: &mut Session, ) -> Result<(), Self::Error> { - if !self + if self .channels .get(&channel) .is_some_and(|state| state.main_attached) { + let signal = match signal { + Sig::HUP => Some(nix::sys::signal::Signal::SIGHUP), + Sig::INT => Some(nix::sys::signal::Signal::SIGINT), + Sig::KILL => Some(nix::sys::signal::Signal::SIGKILL), + Sig::QUIT => Some(nix::sys::signal::Signal::SIGQUIT), + Sig::TERM => Some(nix::sys::signal::Signal::SIGTERM), + _ => None, + }; + if let (Some(signal), Some(main_session)) = (signal, self.main_session.as_ref()) + && let Err(error) = main_session.signal_group(signal) + { + warn!(%error, ?signal, "failed to signal canonical main process group"); + } return Ok(()); } + let Some(process) = self + .channels + .get(&channel) + .and_then(|state| state.process.clone()) + else { + return Ok(()); + }; let signal = match signal { - Sig::HUP => Some(nix::sys::signal::Signal::SIGHUP), - Sig::INT => Some(nix::sys::signal::Signal::SIGINT), - Sig::KILL => Some(nix::sys::signal::Signal::SIGKILL), - Sig::QUIT => Some(nix::sys::signal::Signal::SIGQUIT), - Sig::TERM => Some(nix::sys::signal::Signal::SIGTERM), + Sig::HUP => Some(openshell_isolation_interface::contract::BoundarySignal::Hup), + Sig::INT => Some(openshell_isolation_interface::contract::BoundarySignal::Int), + Sig::KILL => Some(openshell_isolation_interface::contract::BoundarySignal::Kill), + Sig::TERM => Some(openshell_isolation_interface::contract::BoundarySignal::Term), _ => None, }; if let Some(signal) = signal - && let Err(error) = self.main_session.signal_group(signal) + && let Err(error) = process.signal(signal).await { - warn!(%error, ?signal, "failed to signal canonical main process group"); + warn!(%error, ?signal, "failed to signal boundary exec process"); } Ok(()) } } -async fn send_main_output(handle: &Handle, channel: ChannelId, event: MainOutput) -> bool { - match event { - MainOutput::Stdout(data) => handle.data(channel, data).await.is_ok(), - MainOutput::Stderr(data) => handle.extended_data(channel, 1, data).await.is_ok(), - MainOutput::Exit(code) => { - let eof_sent = handle.eof(channel).await.is_ok(); - let status_sent = handle - .exit_status_request(channel, code.max(0).unsigned_abs()) +impl SshHandler { + async fn start_shell( + &mut self, + channel: ChannelId, + handle: Handle, + command: Option, + ) -> anyhow::Result<()> { + let state = self + .channels + .get_mut(&channel) + .ok_or_else(|| anyhow::anyhow!("start_shell on unknown channel {channel:?}"))?; + let no_login_shell = state.no_login_shell; + let pty = state.pty_request.take(); + let pty_requested = pty.is_some(); + let (program, args) = command.map_or_else( + || { + if pty_requested { + ("/bin/bash".to_string(), vec!["-i".to_string()]) + } else { + ("/bin/bash".to_string(), vec![]) + } + }, + |command| { + ( + "/bin/bash".to_string(), + vec![login_shell_flag(no_login_shell).to_string(), command], + ) + }, + ); + let env = pty + .as_ref() + .map(|request| vec![("TERM".to_string(), request.term.clone())]) + .unwrap_or_default(); + self.start_exec_spec( + channel, + handle, + openshell_isolation_interface::contract::ExecSpec { + program, + args, + env, + workdir: None, + pty: pty_requested, + }, + ) + .await?; + if let (Some(pty), Some(terminal)) = ( + pty, + self.channels + .get(&channel) + .and_then(|state| state.terminal.as_ref()), + ) { + terminal + .resize(to_u16(pty.col_width.max(1)), to_u16(pty.row_height.max(1))) .await - .is_ok(); - let close_sent = handle.close(channel).await.is_ok(); - eof_sent && status_sent && close_sent + .map_err(|error| anyhow::anyhow!(error.to_string()))?; } + Ok(()) + } + + async fn start_exec_spec( + &mut self, + channel: ChannelId, + handle: Handle, + spec: openshell_isolation_interface::contract::ExecSpec, + ) -> anyhow::Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let mut exec = self + .boundary_exec + .exec(spec) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; + let state = self + .channels + .get_mut(&channel) + .ok_or_else(|| anyhow::anyhow!("exec on unknown channel {channel:?}"))?; + state.process = Some(exec.process.clone()); + state.terminal = exec.terminal.take(); + + if let Some(mut stdin) = exec.stdin.take() { + let (sender, receiver) = mpsc::channel::>(); + let runtime = tokio::runtime::Handle::current(); + std::thread::spawn(move || { + while let Ok(bytes) = receiver.recv() { + if runtime.block_on(stdin.write_all(&bytes)).is_err() { + break; + } + } + }); + state.input_sender = Some(InputSender::Process(sender)); + } + + let mut stdout = exec.stdout; + let stdout_handle = handle.clone(); + let stdout_task = tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(size) => { + let _ = stdout_handle.data(channel, buffer[..size].to_vec()).await; + } + } + } + }); + let stderr_task = exec.stderr.map(|mut stderr| { + let stderr_handle = handle.clone(); + tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(size) => { + let _ = stderr_handle + .extended_data(channel, 1, buffer[..size].to_vec()) + .await; + } + } + } + }) + }); + tokio::spawn(async move { + let status = exec.process.wait().await; + let _ = stdout_task.await; + if let Some(task) = stderr_task { + let _ = task.await; + } + let code = match status { + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code)) => { + code.max(0).cast_unsigned() + } + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Signaled( + signal, + )) => (128_i32.saturating_add(signal)).max(0).cast_unsigned(), + Err(_) => 1, + }; + let _ = handle.eof(channel).await; + let _ = handle.exit_status_request(channel, code).await; + let _ = handle.close(channel).await; + }); + Ok(()) } -} -impl SshHandler { async fn close_main_attachment( &mut self, channel: ChannelId, @@ -1033,11 +937,15 @@ impl SshHandler { ) { if let Some(state) = self.channels.get_mut(&channel) { if state.main_attached { - self.main_session.end_terminal_attachment(); + if let Some(main_session) = self.main_session.as_ref() { + main_session.end_terminal_attachment(); + } state.main_attached = false; } - if let Some(owner) = state.main_input_owner.take() { - self.main_session.release_input(owner); + if let Some(owner) = state.main_input_owner.take() + && let Some(main_session) = self.main_session.as_ref() + { + main_session.release_input(owner); } state.input_sender.take(); state.main_detach_prefix_pending = false; @@ -1058,62 +966,21 @@ impl SshHandler { let _ = handle.exit_status_request(channel, 0).await; let _ = handle.close(channel).await; } +} - fn start_shell( - &mut self, - channel: ChannelId, - handle: Handle, - command: Option, - ) -> anyhow::Result<()> { - let provider_env = self.provider_credentials.child_env_with_gcp_resolved(); - let state = self - .channels - .get_mut(&channel) - .ok_or_else(|| anyhow::anyhow!("start_shell on unknown channel {channel:?}"))?; - let no_login_shell = state.no_login_shell; - if let Some(pty) = state.pty_request.take() { - // PTY was requested — allocate a real PTY (interactive shell or - // exec that explicitly asked for a terminal). - let (pty_master, input_sender) = spawn_pty_shell( - &self.policy, - &self.workspace, - command, - no_login_shell, - &pty, - handle, - channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &provider_env, - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - state.pty_master = Some(pty_master); - state.input_sender = Some(InputSender::Process(input_sender)); - } else { - // No PTY requested — use plain pipes so stdout/stderr are - // separate and output has clean LF line endings. This is the - // path VSCode Remote-SSH exec commands take. - let input_sender = spawn_pipe_exec( - &self.policy, - &self.workspace, - command, - no_login_shell, - handle, - channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &provider_env, - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - state.input_sender = Some(InputSender::Process(input_sender)); +async fn send_main_output(handle: &Handle, channel: ChannelId, event: MainOutput) -> bool { + match event { + MainOutput::Stdout(data) => handle.data(channel, data).await.is_ok(), + MainOutput::Stderr(data) => handle.extended_data(channel, 1, data).await.is_ok(), + MainOutput::Exit(code) => { + let eof = handle.eof(channel).await.is_ok(); + let status = handle + .exit_status_request(channel, code.max(0).unsigned_abs()) + .await + .is_ok(); + let close = handle.close(channel).await.is_ok(); + eof && status && close } - Ok(()) } } @@ -1131,12 +998,11 @@ impl SshHandler { /// thread could be reused for unrelated tasks and must not be contaminated. /// On non-Linux platforms (no network namespace support), we connect directly. pub async fn connect_in_netns( - addr: &str, - netns_fd: Option, + addr: std::net::SocketAddr, + netns_fd: Option>, ) -> std::io::Result { #[cfg(target_os = "linux")] if let Some(fd) = netns_fd { - let addr = addr.to_string(); let (tx, rx) = tokio::sync::oneshot::channel(); std::thread::spawn(move || { let result = (|| -> std::io::Result { @@ -1144,11 +1010,11 @@ pub async fn connect_in_netns( // SAFETY: setns is safe to call; this is a dedicated thread that // will exit after the connection is established. #[allow(unsafe_code)] - let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; + let rc = unsafe { libc::setns(fd.as_raw_fd(), libc::CLONE_NEWNET) }; if rc != 0 { return Err(std::io::Error::last_os_error()); } - std::net::TcpStream::connect(&addr) + std::net::TcpStream::connect_timeout(&addr, Duration::from_secs(5)) })(); let _ = tx.send(result); }); @@ -1165,11 +1031,19 @@ pub async fn connect_in_netns( #[cfg(not(target_os = "linux"))] let _ = netns_fd; - let stream = tokio::net::TcpStream::connect(addr).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) + tokio::time::timeout( + Duration::from_secs(5), + connect_tcp_nodelay_best_effort(&[addr]), + ) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "connect timed out"))? } +const fn login_shell_flag(no_login_shell: bool) -> &'static str { + if no_login_shell { "-c" } else { "-lc" } +} + +#[allow(dead_code)] #[derive(Clone)] struct PtyRequest { term: String, @@ -1191,6 +1065,35 @@ impl Default for PtyRequest { } } +/// Derive the session USER and HOME from the policy's `run_as_user`. +/// +/// For name-based identities, looks up the home directory via `/etc/passwd` +/// (or defaults to `/home/{user}`). +/// +/// For numeric UIDs, there is no passwd entry — falls back to +/// `("{uid}", "/sandbox")` so the agent session still has a meaningful +/// USER identifier. +pub(crate) fn session_user_and_home(policy: &SandboxPolicy) -> (String, String) { + match policy.process.run_as_user.as_deref() { + Some(user) if !user.is_empty() => { + // Numeric UID — no passwd entry expected; use default HOME. + if user.parse::().is_ok() { + return (user.to_string(), "/sandbox".to_string()); + } + // Name-based identity — look up home from /etc/passwd. + let home = nix::unistd::User::from_name(user) + .ok() + .flatten() + .map_or_else( + || format!("/home/{user}"), + |u| u.dir.to_string_lossy().into_owned(), + ); + (user.to_string(), home) + } + _ => ("sandbox".to_string(), "/sandbox".to_string()), + } +} + #[allow(clippy::too_many_arguments)] pub(crate) fn apply_child_env( cmd: &mut Command, @@ -1238,16 +1141,12 @@ pub(crate) fn apply_child_env( } } -const fn login_shell_flag(no_login_shell: bool) -> &'static str { - if no_login_shell { "-c" } else { "-lc" } -} - #[allow(clippy::too_many_arguments)] +#[allow(dead_code)] fn spawn_pty_shell( policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, + workdir: Option, command: Option, - no_login_shell: bool, pty: &PtyRequest, handle: Handle, channel: ChannelId, @@ -1284,7 +1183,7 @@ fn spawn_pty_shell( }, |command| { let mut c = Command::new("/bin/bash"); - c.arg(login_shell_flag(no_login_shell)).arg(command); + c.arg("-lc").arg(command); c }, ); @@ -1297,7 +1196,7 @@ fn spawn_pty_shell( // Derive USER and HOME from the policy's run_as_user when available, // falling back to "sandbox" / "/sandbox" for backward compatibility. - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); + let (session_user, session_home) = session_user_and_home(policy); apply_child_env( &mut cmd, &session_home, @@ -1310,20 +1209,20 @@ fn spawn_pty_shell( ); cmd.stdin(stdin).stdout(stdout).stderr(stderr); - if let Some(dir) = workspace.root() { + if let Some(dir) = workdir.as_deref() { cmd.current_dir(dir); } // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); } // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workspace.root(), enforcement_mode) + crate::process::prepare_child_sandbox(policy, workdir.as_deref(), enforcement_mode) .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; #[cfg(unix)] @@ -1331,16 +1230,18 @@ fn spawn_pty_shell( unsafe_pty::install_pre_exec( &mut cmd, policy.clone(), - workspace.owned_root(), + workdir.clone(), slave_fd, netns_fd, resolved_identity, enforcement_mode, #[cfg(target_os = "linux")] prepared_sandbox, - ); + )?; } + #[cfg(target_os = "linux")] + let mut child_registry = managed_children::lock(); #[cfg(target_os = "linux")] let mut child = crate::process::spawn_std_command_with_supervisor_identity_namespace(cmd)?; #[cfg(not(target_os = "linux"))] @@ -1348,7 +1249,9 @@ fn spawn_pty_shell( #[cfg(target_os = "linux")] let child_pid = child.id(); #[cfg(target_os = "linux")] - let managed_child = managed_children::register(child_pid); + let managed_child = child_registry.register(child_pid); + #[cfg(target_os = "linux")] + drop(child_registry); let master_file = master; let (sender, receiver) = mpsc::channel::>(); @@ -1394,8 +1297,8 @@ fn spawn_pty_shell( std::thread::spawn(move || { let status = child.wait().ok(); #[cfg(target_os = "linux")] - if let Some(child) = managed_child { - managed_children::unregister(child); + if let Some(managed_child) = managed_child { + managed_children::unregister(managed_child); } let code = status.and_then(|s| s.code()).unwrap_or(1).unsigned_abs(); // Wait for the reader thread to finish forwarding all output before @@ -1422,11 +1325,11 @@ fn spawn_pty_shell( /// (type 1), preserving the separation that clients like `VSCode` Remote-SSH /// expect. Output retains clean LF line endings (no CRLF translation). #[allow(clippy::too_many_arguments)] +#[allow(dead_code)] fn spawn_pipe_exec( policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, + workdir: Option, command: Option, - no_login_shell: bool, handle: Handle, channel: ChannelId, netns_fd: Option, @@ -1449,15 +1352,15 @@ fn spawn_pipe_exec( }, |command| { let mut c = Command::new("/bin/bash"); - // Login shell (-l) sources .profile/.bashrc so tool env vars - // (VIRTUAL_ENV, etc.) are available. Callers that need a predictable - // environment opt out via OPENSHELL_NO_LOGIN_SHELL → plain -c. - c.arg(login_shell_flag(no_login_shell)).arg(command); + // Use login shell (-l) so that .profile/.bashrc are sourced and + // tool-specific env vars (VIRTUAL_ENV, UV_PYTHON_INSTALL_DIR, etc.) + // are available without hardcoding them here. + c.arg("-lc").arg(command); c }, ); - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); + let (session_user, session_home) = session_user_and_home(policy); apply_child_env( &mut cmd, &session_home, @@ -1472,20 +1375,20 @@ fn spawn_pipe_exec( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - if let Some(dir) = workspace.root() { + if let Some(dir) = workdir.as_deref() { cmd.current_dir(dir); } // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); } // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workspace.root(), enforcement_mode) + crate::process::prepare_child_sandbox(policy, workdir.as_deref(), enforcement_mode) .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; #[cfg(unix)] @@ -1493,15 +1396,17 @@ fn spawn_pipe_exec( unsafe_pty::install_pre_exec_no_pty( &mut cmd, policy.clone(), - workspace.owned_root(), + workdir.clone(), netns_fd, resolved_identity, enforcement_mode, #[cfg(target_os = "linux")] prepared_sandbox, - ); + )?; } + #[cfg(target_os = "linux")] + let mut child_registry = managed_children::lock(); #[cfg(target_os = "linux")] let mut child = crate::process::spawn_std_command_with_supervisor_identity_namespace(cmd)?; #[cfg(not(target_os = "linux"))] @@ -1509,7 +1414,9 @@ fn spawn_pipe_exec( #[cfg(target_os = "linux")] let child_pid = child.id(); #[cfg(target_os = "linux")] - let managed_child = managed_children::register(child_pid); + let managed_child = child_registry.register(child_pid); + #[cfg(target_os = "linux")] + drop(child_registry); let child_stdin = child.stdin.take(); let child_stdout = child.stdout.take().expect("stdout must be piped"); @@ -1581,8 +1488,8 @@ fn spawn_pipe_exec( std::thread::spawn(move || { let status = child.wait().ok(); #[cfg(target_os = "linux")] - if let Some(child) = managed_child { - managed_children::unregister(child); + if let Some(managed_child) = managed_child { + managed_children::unregister(managed_child); } let code = status.and_then(|s| s.code()).unwrap_or(1).unsigned_abs(); // Wait for both reader threads. @@ -1648,6 +1555,10 @@ pub(crate) mod unsafe_pty { #[allow(unsafe_code)] #[allow(clippy::too_many_arguments)] + #[allow( + clippy::unnecessary_wraps, + reason = "keeps pre-exec installation fallible for callers if setup gains validation" + )] #[cfg_attr( not(target_os = "linux"), allow( @@ -1664,7 +1575,7 @@ pub(crate) mod unsafe_pty { resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, #[cfg(target_os = "linux")] prepared: Option, - ) { + ) -> anyhow::Result<()> { // Wrap in Option so we can .take() it out of the FnMut closure. // pre_exec is only called once (after fork, before exec). #[cfg(target_os = "linux")] @@ -1684,12 +1595,17 @@ pub(crate) mod unsafe_pty { ) }); } + Ok(()) } /// Pre-exec hook for pipe-based (non-PTY) exec. /// /// Skips `setsid` and `TIOCSCTTY` since there is no controlling terminal. #[allow(unsafe_code)] + #[allow( + clippy::unnecessary_wraps, + reason = "keeps pre-exec installation fallible for callers if setup gains validation" + )] #[cfg_attr( not(target_os = "linux"), allow( @@ -1705,11 +1621,14 @@ pub(crate) mod unsafe_pty { resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, #[cfg(target_os = "linux")] prepared: Option, - ) { + ) -> anyhow::Result<()> { #[cfg(target_os = "linux")] let mut prepared = prepared; unsafe { cmd.pre_exec(move || { + if libc::setpgid(0, 0) != 0 { + return Err(std::io::Error::last_os_error()); + } enter_netns_and_sandbox( netns_fd, &policy, @@ -1720,6 +1639,7 @@ pub(crate) mod unsafe_pty { ) }); } + Ok(()) } fn enter_netns_and_sandbox( @@ -1810,6 +1730,22 @@ fn is_loopback_host(host: &str) -> bool { } } +/// Resolve a (loopback-validated) destination host string to an `IpAddr`, +/// mapping `localhost` to `127.0.0.1`. +/// +/// Returns `None` for anything that does not parse to an IP, so +/// [`LoopbackTarget::new`] never sees a hostname. +fn loopback_ip(host: &str) -> Option { + let host = host + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(host); + if host.eq_ignore_ascii_case("localhost") { + return Some(std::net::Ipv4Addr::LOCALHOST.into()); + } + host.parse().ok() +} + #[cfg(test)] #[allow( clippy::doc_markdown, @@ -1820,20 +1756,6 @@ mod tests { use super::*; use std::process::Stdio; - /// Regression test: the direct-tcpip connect path sets `TCP_NODELAY`. - #[tokio::test] - async fn connect_in_netns_sets_tcp_nodelay() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind listener"); - let addr = listener.local_addr().expect("local addr"); - - let stream = connect_in_netns(&addr.to_string(), None) - .await - .expect("connect"); - assert!(stream.nodelay().expect("query TCP_NODELAY")); - } - #[cfg(unix)] fn file_mode(path: &Path) -> u32 { use std::os::unix::fs::PermissionsExt; @@ -1950,38 +1872,6 @@ mod tests { assert_eq!(output.stdout, b"hello"); } - /// Command execution selects a login shell by default and a non-login shell - /// under `--no-login-shell`, so user startup files are sourced only in the - /// default case. - #[cfg(unix)] - #[test] - fn login_shell_flag_controls_profile_sourcing() { - let home = tempfile::tempdir().unwrap(); - std::fs::write(home.path().join(".bash_profile"), "echo LOGIN_MARKER\n").unwrap(); - - let run = |flag: &str| -> String { - let out = Command::new("bash") - .arg(flag) - .arg("true") - .env("HOME", home.path()) - .env_remove("BASH_ENV") // isolate: -c still reads BASH_ENV if set - .output() - .expect("spawn bash"); - String::from_utf8_lossy(&out.stdout).into_owned() - }; - - assert_eq!(login_shell_flag(true), "-c"); - assert_eq!(login_shell_flag(false), "-lc"); - assert!( - run("-lc").contains("LOGIN_MARKER"), - "login shell must source .bash_profile" - ); - assert!( - !run("-c").contains("LOGIN_MARKER"), - "non-login shell must not source it" - ); - } - /// Verify that the stdin writer delivers all buffered data before exiting /// when the sender is dropped. This ensures channel_eof doesn't cause /// data loss — only signals "no more data after this". @@ -2191,56 +2081,6 @@ mod tests { assert_eq!(rx_b.recv().unwrap(), b"still-alive"); } - #[test] - fn main_detach_filter_forwards_ctrl_c_unchanged() { - let mut prefix_pending = false; - let (forward, detach) = - filter_main_detach_sequence(&mut prefix_pending, b"before\x03after"); - - assert_eq!(forward, b"before\x03after"); - assert!(!detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_removes_sequence_and_trailing_input() { - let mut prefix_pending = false; - let (forward, detach) = - filter_main_detach_sequence(&mut prefix_pending, b"before\x10\x11after"); - - assert_eq!(forward, b"before"); - assert!(detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_recognizes_sequence_across_frames() { - let mut prefix_pending = false; - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"before\x10"); - assert_eq!(forward, b"before"); - assert!(!detach); - assert!(prefix_pending); - - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"\x11"); - assert!(forward.is_empty()); - assert!(detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_forwards_unmatched_prefix() { - let mut prefix_pending = false; - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"\x10"); - assert!(forward.is_empty()); - assert!(!detach); - assert!(prefix_pending); - - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"x"); - assert_eq!(forward, b"\x10x"); - assert!(!detach); - assert!(!prefix_pending); - } - // ----------------------------------------------------------------------- // session_user_and_home tests (Phase 2: numeric UID support) // ----------------------------------------------------------------------- @@ -2260,33 +2100,12 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "1000"); // Numeric UID has no passwd entry — defaults to /sandbox. assert_eq!(home, "/sandbox"); } - #[test] - fn session_user_and_home_uses_driver_workspace_when_supplied() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("1234".into()), - run_as_group: Some("1235".into()), - }, - }; - - let (user, home) = session_user_and_home(&policy, Some("/workspace/project")); - assert_eq!(user, "1234"); - assert_eq!(home, "/workspace/project"); - } - #[test] fn session_user_and_home_returns_name_from_passwd() { use openshell_core::policy::{ @@ -2302,7 +2121,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "sandbox"); // Name-based — should resolve via passwd (or /home/{user}). assert!(!home.is_empty()); @@ -2323,7 +2142,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "sandbox"); assert_eq!(home, "/sandbox"); } @@ -2343,7 +2162,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "sandbox"); assert_eq!(home, "/sandbox"); } @@ -2363,7 +2182,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "1000660000"); assert_eq!(home, "/sandbox"); } @@ -2427,7 +2246,8 @@ mod tests { ) .expect("prepare should succeed in test environment"), ), - ); + ) + .expect("install pre_exec should succeed"); let output = cmd .spawn() @@ -2482,7 +2302,8 @@ mod tests { ProcessEnforcementMode::Full, #[cfg(target_os = "linux")] None, - ); + ) + .expect("install pre_exec should succeed"); let output = cmd .spawn() @@ -2495,299 +2316,4 @@ mod tests { "resolved-identity-ok" ); } - - // ----------------------------------------------------------------------- - // direct-tcpip authorization wiring (SEC-007) - // - // The `loopback_host_*` tests above cover the predicate in isolation. - // These drive the real `russh::server::Handler` over an in-memory duplex - // so the deny path itself is covered: channel-open authorization travels - // through a reply handle rather than the handler's return value, so a - // handler that never rejects anything still type-checks and still passes - // every predicate test. - // ----------------------------------------------------------------------- - - struct AcceptAnyServerKey; - - impl russh::client::Handler for AcceptAnyServerKey { - type Error = russh::Error; - - async fn check_server_key( - &mut self, - _server_public_key: &russh::keys::PublicKey, - ) -> Result { - Ok(true) - } - } - - fn forwarding_test_policy() -> SandboxPolicy { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - - SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - } - } - - /// Serve `SshHandler` on one end of an in-memory duplex and return an - /// authenticated client handle for the other end. - /// - /// The handler gets `netns_fd: None` so `connect_in_netns` performs a plain - /// TCP connect, making the forwarding path reachable without a network - /// namespace. - async fn authenticated_test_client_with_main( - main_session: Arc, - ) -> russh::client::Handle { - // Scoped so the `!Send` ThreadRng is dropped before the first await. - let host_key = { - let mut rng = rand::rng(); - PrivateKey::random(&mut rng, Algorithm::Ed25519).expect("host key") - }; - let mut server_config = russh::server::Config { - auth_rejection_time: Duration::from_millis(1), - ..Default::default() - }; - server_config.keys.push(host_key); - - let handler = SshHandler::new( - forwarding_test_policy(), - ResolvedWorkspace::default(), - None, - None, - None, - ProviderCredentialState::from_child_env_snapshot(0, HashMap::new()), - HashMap::new(), - ResolvedProcessIdentity::default(), - ProcessEnforcementMode::NetworkOnly, - main_session, - ); - - let (server_stream, client_stream) = tokio::io::duplex(64 * 1024); - tokio::spawn(async move { - if let Ok(session) = - russh::server::run_stream(Arc::new(server_config), server_stream, handler).await - { - let _ = session.await; - } - }); - - let mut client = russh::client::connect_stream( - Arc::new(russh::client::Config::default()), - client_stream, - AcceptAnyServerKey, - ) - .await - .expect("SSH handshake should complete over the duplex"); - - let auth = client - .authenticate_none("sandbox") - .await - .expect("auth_none should not error"); - assert!( - matches!(auth, russh::client::AuthResult::Success), - "sandbox SSH server accepts the none auth method" - ); - - client - } - - async fn authenticated_test_client() -> russh::client::Handle { - authenticated_test_client_with_main(MainSession::inert()).await - } - - #[tokio::test] - async fn abrupt_transport_drop_releases_main_input_lease() { - let main_session = MainSession::inert(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let channel = client.channel_open_session().await.expect("open session"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - match main_session.acquire_input() { - Err(_) => break, - Ok((owner, _)) => main_session.release_input(owner), - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should acquire canonical input lease"); - - drop(channel); - drop(client); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if main_session.acquire_input().is_ok() { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("handler drop should release canonical input lease"); - } - - #[tokio::test] - async fn main_attachment_closes_naturally_after_terminal_delivery() { - let main_session = MainSession::inert(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let mut channel = client.channel_open_session().await.expect("open session"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - match main_session.acquire_input() { - Err(_) => break, - Ok((owner, _)) => main_session.release_input(owner), - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should register its attachment"); - - assert!(main_session.finish(7, false).await); - main_session.mark_terminal_reported(); - - let exit_status = tokio::time::timeout(Duration::from_secs(1), async { - let mut exit_status = None; - loop { - match channel.wait().await { - Some(russh::ChannelMsg::ExitStatus { - exit_status: status, - }) => { - exit_status = Some(status); - } - Some(russh::ChannelMsg::Close) => break exit_status, - None => panic!("main channel ended without a close message"), - Some(_) => {} - } - } - }) - .await - .expect("main channel should deliver its exit status"); - assert_eq!(exit_status, Some(7)); - drop(channel); - drop(client); - - tokio::time::timeout( - Duration::from_secs(1), - main_session.wait_for_terminal_attachments(), - ) - .await - .expect("peer channel close should release terminal delivery"); - } - - #[tokio::test] - async fn main_subsystem_applies_initial_pty_dimensions() { - let (main_session, _slave) = MainSession::terminal_for_test(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let channel = client.channel_open_session().await.expect("open session"); - channel - .request_pty(true, "xterm-256color", 200, 60, 1600, 900, &[]) - .await - .expect("request PTY"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if main_session.terminal_size_for_test() == (200, 60) { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should apply the initial PTY dimensions"); - } - - #[tokio::test] - async fn direct_tcpip_rejects_non_loopback_destination() { - let client = authenticated_test_client().await; - - let err = client - .channel_open_direct_tcpip("10.0.0.1", 80, "127.0.0.1", 0) - .await - .expect_err("forwarding to a non-loopback host must be refused"); - - assert!( - matches!( - err, - russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) - ), - "expected AdministrativelyProhibited, got {err:?}" - ); - } - - #[tokio::test] - async fn direct_tcpip_rejects_port_above_tcp_range() { - let client = authenticated_test_client().await; - - // 65_537 truncates to port 1 when cast to u16, so the guard has to - // reject it before the cast rather than forward to a privileged port. - let err = client - .channel_open_direct_tcpip("127.0.0.1", 65_537, "127.0.0.1", 0) - .await - .expect_err("a port outside the TCP range must be refused"); - - assert!( - matches!( - err, - russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) - ), - "expected AdministrativelyProhibited, got {err:?}" - ); - } - - #[tokio::test] - async fn direct_tcpip_forwards_to_loopback_listener() { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind loopback echo listener"); - let port = listener.local_addr().expect("listener address").port(); - tokio::spawn(async move { - if let Ok((mut socket, _)) = listener.accept().await { - let mut buf = [0u8; 64]; - if let Ok(n) = socket.read(&mut buf).await - && n > 0 - { - let _ = socket.write_all(&buf[..n]).await; - } - } - }); - - let client = authenticated_test_client().await; - let channel = client - .channel_open_direct_tcpip("127.0.0.1", u32::from(port), "127.0.0.1", 0) - .await - .expect("forwarding to a loopback listener must be allowed"); - - let mut stream = channel.into_stream(); - stream.write_all(b"ping").await.expect("write to channel"); - - let mut echoed = [0u8; 4]; - tokio::time::timeout(Duration::from_secs(10), stream.read_exact(&mut echoed)) - .await - .expect("relayed response should arrive before the timeout") - .expect("read from channel"); - assert_eq!(&echoed, b"ping", "bytes round-trip through the tunnel"); - } } diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 98a3c0497b..db01d07323 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -11,8 +11,6 @@ //! selection — it has no protocol awareness of the bytes flowing through. use std::net::IpAddr; -#[cfg(target_os = "linux")] -use std::os::fd::RawFd; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; @@ -23,6 +21,7 @@ use openshell_core::proto::{ RelayOpenResult, ReportMainProcessExitRequest, SupervisorHeartbeat, SupervisorHello, SupervisorMessage, TcpRelayTarget, gateway_message, relay_open, supervisor_message, }; +use openshell_isolation_interface::contract::{BoundaryPortForward, LoopbackTarget}; use openshell_ocsf::{ ActivityId, ConnectionInfo, Endpoint, NetworkActivityBuilder, OcsfEvent, SandboxContext, SeverityId, StatusId, ocsf_emit, @@ -33,7 +32,6 @@ use tokio_stream::StreamExt; use tracing::{debug, warn}; use openshell_core::grpc_client; -use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::transport_errors::is_expected_transport_close_status; const INITIAL_BACKOFF: Duration = Duration::from_secs(1); @@ -278,7 +276,7 @@ pub fn spawn( endpoint: String, sandbox_id: String, ssh_socket_path: std::path::PathBuf, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, instance_id: String, @@ -287,7 +285,7 @@ pub fn spawn( endpoint, sandbox_id, ssh_socket_path, - netns_fd, + port_forward, expected_ssh_peer_pid, terminating, instance_id, @@ -299,7 +297,7 @@ struct SessionConfig { endpoint: String, sandbox_id: String, ssh_socket_path: std::path::PathBuf, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, instance_id: String, @@ -392,6 +390,7 @@ async fn run_single_session( heartbeat_secs, ); ocsf_emit!(event); + // Main loop: receive gateway messages + send heartbeats. let mut heartbeat_interval = tokio::time::interval(Duration::from_secs(u64::from(heartbeat_secs))); @@ -411,7 +410,7 @@ async fn run_single_session( let context = GatewayMessageContext { sandbox_id: &config.sandbox_id, ssh_socket_path: &config.ssh_socket_path, - netns_fd: config.netns_fd, + port_forward: &config.port_forward, expected_ssh_peer_pid: config.expected_ssh_peer_pid, channel: &channel, tx: &tx, @@ -479,7 +478,7 @@ pub async fn finalize_main_process_exit( struct GatewayMessageContext<'a> { sandbox_id: &'a str, ssh_socket_path: &'a std::path::Path, - netns_fd: Option, + port_forward: &'a Arc, expected_ssh_peer_pid: Option, channel: &'a grpc_client::AuthedChannel, tx: &'a mpsc::Sender, @@ -498,7 +497,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< let channel = context.channel.clone(); let ssh_socket_path = context.ssh_socket_path.to_path_buf(); let tx = context.tx.clone(); - let netns_fd = context.netns_fd; + let port_forward = context.port_forward.clone(); let expected_ssh_peer_pid = context.expected_ssh_peer_pid; let terminating = Arc::clone(context.terminating); @@ -510,7 +509,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< match handle_relay_open( relay_open, &ssh_socket_path, - netns_fd, + port_forward, expected_ssh_peer_pid, channel, tx, @@ -567,7 +566,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< async fn handle_relay_open( relay_open: RelayOpen, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, channel: grpc_client::AuthedChannel, tx: mpsc::Sender, @@ -577,7 +576,7 @@ async fn handle_relay_open( let target = match open_target( &relay_open, ssh_socket_path, - netns_fd, + &port_forward, expected_ssh_peer_pid, ) .await @@ -722,11 +721,11 @@ async fn send_relay_open_result( async fn open_target( relay_open: &RelayOpen, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: &Arc, expected_ssh_peer_pid: Option, ) -> Result, Box> { match relay_open.target.as_ref() { - Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, netns_fd).await, + Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, port_forward).await, Some(relay_open::Target::Ssh(_)) | None => { let runtime_path = crate::unix_socket::runtime_path(ssh_socket_path); let stream = tokio::net::UnixStream::connect(runtime_path.as_ref()).await?; @@ -747,59 +746,27 @@ async fn open_target( async fn open_tcp_target( target: &TcpRelayTarget, - netns_fd: Option, + port_forward: &Arc, ) -> Result, Box> { let host = normalize_tcp_target_host(target)?; let port = u16::try_from(target.port).map_err(|_| "tcp target port must fit in u16")?; - let stream = connect_tcp_target(host, port, netns_fd).await?; + // `normalize_tcp_target_host` returns a loopback IP string; parse it and let + // `LoopbackTarget::new` re-validate before connecting. + let ip: IpAddr = host + .parse() + .map_err(|_| "tcp target host must be a loopback IP")?; + let target = LoopbackTarget::new(ip, port) + .map_err(|e| -> Box { e.to_string().into() })?; + // Connect inside the boundary through the injected port-forward interface + // (RFC 0012). In-pod this enters the workload netns, a delegated backend + // tunnels into its guest. + let stream = port_forward + .connect(target) + .await + .map_err(|e| -> Box { e.to_string().into() })?; Ok(Box::new(stream)) } -#[cfg(target_os = "linux")] -async fn connect_tcp_target( - host: String, - port: u16, - netns_fd: Option, -) -> Result> { - if let Some(fd) = netns_fd { - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result { - #[allow(unsafe_code)] - let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - std::net::TcpStream::connect((host.as_str(), port)) - })(); - let _ = tx.send(result); - }); - - let stream = rx - .await - .map_err(|_| "netns tcp connect thread panicked")??; - stream.set_nonblocking(true)?; - let stream = tokio::net::TcpStream::from_std(stream)?; - set_tcp_nodelay_best_effort(&stream); - return Ok(stream); - } - - let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) -} - -#[cfg(not(target_os = "linux"))] -async fn connect_tcp_target( - host: String, - port: u16, - _netns_fd: Option, -) -> Result> { - let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) -} - #[cfg(test)] fn validate_tcp_target(target: &TcpRelayTarget) -> Result<(), String> { normalize_tcp_target_host(target).map(|_| ()) @@ -839,20 +806,6 @@ mod target_tests { } } - /// Regression test: the TCP relay connect path sets `TCP_NODELAY`. - #[tokio::test] - async fn connect_tcp_target_sets_tcp_nodelay() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind listener"); - let addr = listener.local_addr().expect("local addr"); - - let stream = connect_tcp_target(addr.ip().to_string(), addr.port(), None) - .await - .expect("connect"); - assert!(stream.nodelay().expect("query TCP_NODELAY")); - } - #[test] fn tcp_target_allows_loopback_hosts() { validate_tcp_target(&tcp("127.0.0.1", 8080)).expect("ipv4 loopback"); @@ -1135,7 +1088,12 @@ mod ocsf_event_tests { }); let relay = ssh_relay_open("peer-check"); - let trusted = open_target(&relay, &socket, None, Some(std::process::id())) + // The SSH relay path does not use the port-forward (that is the TCP + // target path); connect from the supervisor's own namespace. + let port_forward: Arc = + Arc::new(crate::boundary_io::NetnsPortForward::new(None, None)); + + let trusted = open_target(&relay, &socket, &port_forward, Some(std::process::id())) .await .expect("matching peer PID should be accepted"); drop(trusted); @@ -1143,7 +1101,7 @@ mod ocsf_event_tests { let Err(err) = open_target( &relay, &socket, - None, + &port_forward, Some(std::process::id().saturating_add(1)), ) .await diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index d515fd70b1..51512f3aa5 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -6,13 +6,23 @@ # The static sandbox binary is staged at: # deploy/docker/.build/prebuilt-binaries//openshell-sandbox # -# Alpine supplies nftables and iptables for pod-namespace egress enforcement. +# Alpine supplies the trusted helper runtime used by boundary mode for +# namespace setup and egress enforcement. The runtime is copied into workload +# images as a root-owned, read-only directory instead of trusting their tools. FROM alpine:3.22 AS supervisor ARG TARGETARCH -RUN apk add --no-cache nftables iptables iptables-legacy +RUN apk add --no-cache iproute2 nftables iptables iptables-legacy \ + && mkdir -p /openshell-runtime \ + && cp -aL /bin /sbin /lib /usr/bin /usr/sbin /usr/lib /openshell-runtime/ \ + && mkdir -p /openshell-runtime/etc /openshell-runtime/usr/share \ + && if [ -d /etc/iproute2 ]; then cp -aL /etc/iproute2 /openshell-runtime/etc/; fi \ + && if [ -d /usr/share/nftables ]; then cp -aL /usr/share/nftables /openshell-runtime/usr/share/; fi \ + && test -x /openshell-runtime/bin/sh \ + && test -x /openshell-runtime/sbin/ip \ + && test -x /openshell-runtime/sbin/nft # Keep the binary root-owned for Podman image-volume mounts and executable by # the Kubernetes network sidecar's non-root proxy UID. diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md index 2be6b3f6d5..c46b435896 100644 --- a/rfc/0012-isolation-backend/README.md +++ b/rfc/0012-isolation-backend/README.md @@ -19,7 +19,7 @@ links: Today the supervisor both builds the workload's isolation boundary and applies its network policy. Because the supervisor runs inside the agent container, the privilege needed to build that boundary sits beside the code it confines. This RFC moves boundary construction and process operations behind a pluggable **Isolation Backend**. The supervisor continues to apply approved network policy through network mediation. -The compute driver provisions the workload and trusted components. The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. The backend establishes the isolation controls, manages workload processes, and routes egress to network mediation. The same lifecycle supports today's in-pod implementation and future delegated implementations without topology-specific supervisor paths. +The compute driver prepares the workload topology and trusted inputs. The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. OpenShell packages that role as `openshell-sandbox --mode=control`. When the workload is separated by a container, pod, userspace kernel, or VM boundary, the same binary runs a small trusted counterpart as `openshell-sandbox --mode=boundary`. The boundary mode owns process observation and operations that cannot be implemented portably from outside the boundary; it has no gateway credentials and no policy authority. ## Motivation @@ -33,56 +33,67 @@ All three come from coupling boundary construction to boundary operation. A comm ## Non-goals -- **Implementing a delegated backend.** Each topology requires its own design and implementation. +- **Standardizing a topology's resource API.** Docker, Kubernetes, VM, and other resource mechanisms remain backend-specific. - **Changing authorization.** [RFC 0001](../0001-core-architecture/README.md) owns control-plane and sandbox identity. A delegated backend must still authenticate callers and scope them to one boundary. -- **Standardizing backend-internal component coordination.** A backend may coordinate helper, sidecar, or interception processes behind one lifecycle; how those components cooperate is backend-specific, not contract surface. +- **Standardizing resource-specific provisioning or transport setup.** A driver still decides how to place the binary, create a private Unix socket, authenticated TCP endpoint, or vsock endpoint, and establish kernel-specific egress capture. This RFC standardizes the authenticated control-to-boundary messages carried over that endpoint. - **Changing gateway lifecycle or public status.** This RFC adds no gateway activation operation, public phase, or status API, and it does not define how a boundary's effective isolation model is surfaced to operators. ## Proposal The mental model has three roles: -- The **compute driver** provisions the sandbox instance according to the selected placement of the workload and trusted isolation components. That placement is the **topology**. +- The **compute driver** prepares placement and trusted topology inputs and owns durable resource provisioning, deletion, and reconciliation. That placement is the **topology**. - The **Isolation Backend** establishes and operates the topology-specific controls around the workload. It also routes workload egress to network mediation and provides process operations. - The **logical supervisor** is the trusted control-plane bridge between the gateway and the workload. It drives the backend, handles authorized gateway requests, and applies approved network policy through network mediation. Together, network policy, filesystem isolation, syscall filtering, and sandbox identity form the workload's isolation boundary. The roles above enforce that boundary and may run in one process or across several trusted components. Their placement does not change the contract. -Each active boundary has at most one logical supervisor, which may span multiple coupled processes. The backend routes all workload egress through a per-boundary source, and the supervisor consumes that source. Internal delegation and transport remain topology-private. +Each active boundary has exactly one control role and at most one boundary role. Together they implement one logical supervisor; boundary mode is not an independently authorized supervisor. The control role owns the gateway session, admitted policy, network-policy decisions, and RFC 0012 lifecycle. Boundary mode owns boundary-local process groups, `exec`, signal, wait, PTY, loopback forwarding, binary observation, and egress capture. The transport and physical placement remain driver-owned. [RFC 0001](../0001-core-architecture/README.md) continues to own sandbox authentication and authorization. In this contract, sandbox identity means binding the authenticated sandbox context to the isolation boundary. -Admission selects the sandbox's topology and determines its trusted context. The compute driver sets up the topology and gives the logical supervisor a `TopologyDescriptor` describing what it provisioned. The supervisor uses the descriptor to attach the matching Isolation Backend. The backend prepares the required controls before the agent starts. +Admission selects the sandbox's topology and trusted context. The compute driver gives the logical supervisor a `TopologyDescriptor` for the matching backend. Its opaque payload can identify an existing resource or carry trusted prepared inputs from which `attach` establishes the boundary. The backend prepares the required controls before the agent starts. ```mermaid flowchart TB Gateway["Gateway"] -->|"create sandbox"| Driver["Compute driver"] - subgraph Topology["Driver-provisioned topology (placement varies)"] - Supervisor["Supervisor"] - Backend["Isolation Backend (may coordinate components)"] + subgraph Topology["Admitted topology (placement varies)"] + Control["openshell-sandbox
--mode=control"] + Backend["Remote Isolation Backend"] subgraph Boundary["Isolation boundary"] - Mediator["Network mediation"] + BoundaryAgent["openshell-sandbox
--mode=boundary"] subgraph Execution["Workload execution environment"] Workload["Workload"] end end - - Supervisor -->|"drives contract"| Backend - Backend -->|"establishes and confirms"| Boundary - Backend -.->|"routes all workload egress to"| Mediator - Supervisor -.->|"applies network policy through"| Mediator - Backend -->|"after Ready: makes admitted agent runnable"| Workload - Workload ==>|"only egress"| Mediator + Mediator["Network mediation"] + + Control -->|"drives RFC 0012"| Backend + Backend <-->|"authenticated, versioned
boundary protocol"| BoundaryAgent + BoundaryAgent -->|"start / exec / signal / wait"| Workload + Workload ==>|"captured egress"| BoundaryAgent + BoundaryAgent ==>|"attributed streams"| Mediator + Control -.->|"policy decisions"| Mediator end - Driver -->|"resources + TopologyDescriptor"| Supervisor + Driver -->|"resources + protected configs"| BoundaryAgent + Driver -->|"trusted TopologyDescriptor"| Control + Gateway <-->|"authorized session"| Control Mediator -->|"allowed egress"| Egress["Egress"] ``` -In the in-pod topology, the supervisor drives a backend implemented in the same process. Other topologies may delegate backend operations without changing the supervisor lifecycle. +Co-located deployments may keep the existing in-process backend and omit boundary mode. Separated deployments use the shared remote backend and boundary protocol, so adding a driver changes provisioning and transport selection without adding a topology branch to the control role. + +### Supervisor modes and boundary protocol + +`openshell-sandbox --mode=control` runs outside the untrusted execution environment. It receives the admitted backend name and a protected `TopologyDescriptor`, resolves the backend without fallback, and owns the gateway-facing access plane. It is the only mode that possesses gateway credentials or applies approved network policy. + +`openshell-sandbox --mode=boundary --boundary-config ` runs inside, or immediately adjacent to, the execution environment. The driver supplies the config over a protected filesystem channel. It names one boundary, one private listener, one per-boundary bootstrap credential, and the workload identity. The identity is either a platform-resolved numeric UID/GID pair or an OCI `Config.User` declaration that boundary mode resolves against the workload filesystem. Boundary mode authenticates and scopes every request to that boundary. It never accepts policy or identity claims from workload code and it cannot authorize an operation independently of control mode. -A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver owns the sandbox instance and topology lifecycle. +The shared protocol is versioned independently of a driver's resource descriptor. It carries `attach`, `confirm`, `start_agent`, agent and exec `wait`/`signal`/`terminate`, PTY resize, loopback-only forwarding, and attributed egress streams. Drivers may use a private Unix socket, authenticated TCP scoped by the runtime network, a Unix endpoint mapped to guest vsock, or host `AF_VSOCK`; adding a transport does not change lifecycle or operation semantics. Secrets are delivered in protected files, redacted from debug output, and never placed in workload-visible environment or process arguments. When a bind-mounted file's host owner may match the workload UID, the driver requires boundary mode to re-own it as root-only before the first workload instruction. Unix socket inodes permit cross-UID control within their private driver-owned directory; the per-boundary bootstrap credential authenticates every request. + +A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver or external orchestrator owns the durable resource lifecycle even when the backend establishes a resource as part of `attach`. ### Contract invariants @@ -91,7 +102,7 @@ Six invariants hold for every boundary: 1. Workload egress is denied except through network mediation for the boundary's lifetime. 2. No untrusted instruction executes until every admitted control applicable to that process is in force. 3. An operation is authorized only when the complete effective policy permits it; network operations are decided through network mediation. There is no silent weakening. -4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the compute driver's provisioned execution environment. +4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the admitted execution environment. 5. Shared infrastructure preserves strict per-boundary lifecycle, policy, identity, enforcement, and cleanup isolation. 6. If the logical supervisor is lost, the boundary remains under its last confirmed enforcement state while supervisor-dependent operations fail closed. Loss of required enforcement ends `Running` and terminates all workload processes within a documented bound; detection and termination may be performed by a trusted node or control-plane actor. Network-mediation unavailability denies outbound connections and never enables direct egress. @@ -99,21 +110,22 @@ Each backend states its termination bound in its implementation documentation. L ### Provisioning -Provisioning runs on the control plane, and three rules hold in every topology: +Provisioning is selected by trusted control-plane configuration, and four rules hold in every topology: -1. **Admission selects the topology** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. The `TopologyDescriptor` supplied by the compute driver must name that backend, and resolution never falls back to another backend. -2. **The compute driver provisions the topology** and anything the selected backend needs. -3. **The backend establishes standing enforcement before untrusted code runs**, during provisioning or `attach`, depending on the backend. +1. **Admission selects the topology** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. The `TopologyDescriptor` must name that backend, and resolution never falls back to another backend. +2. **The descriptor supports prepared and existing resources.** A compute driver or orchestrator may identify an existing resource, or it may supply trusted prepared inputs that the backend uses to establish the resource during `attach`. +3. **The compute driver owns the durable resource lifecycle.** It provisions or prepares, deletes, and reconciles the topology independently of logical-supervisor availability. +4. **The backend establishes standing enforcement before untrusted code runs**, during `attach` or `confirm`, depending on the backend. If a topology depends on cluster-scoped coverage or registration, admission verifies that the prerequisite covers the boundary's placement before untrusted code runs. Every topology provides a trusted cleanup path that does not depend on logical-supervisor availability. -A compute driver may provision a resource and `TopologyDescriptor` before the control plane assigns it to a sandbox. No untrusted workload runs while the resource is unassigned. After claim or assignment produces a trusted `SandboxContext`, the supervisor calls `attach`; the backend either binds that context to the prepared resource and returns `Bound`, or rejects it as incompatible. Pool creation, claim, reset, release, and recycling remain outside this contract. +A compute driver may provision a resource and `TopologyDescriptor` before the control plane assigns it to a sandbox. No untrusted workload runs while the resource is unassigned. It may instead prepare immutable image or disk identities, normalized runtime settings, placement results, or protected references to artifacts and encode those inputs in the descriptor. After assignment produces a trusted `SandboxContext`, the supervisor calls `attach`; the backend atomically establishes or locates the resource, binds that context, and returns `Bound`, or rejects it as incompatible. Attach-time establishment is idempotent on trusted sandbox identity and launch generation. Partial resources are removed or remain labeled for compute-driver reconciliation. Pool creation, claim, reset, release, and recycling remain outside this contract. ### The topology descriptor -The driver supplies a descriptor for every topology admitted to this contract, including in-pod and resources prepared before assignment. The common envelope names the backend and carries an opaque payload. +The compute driver supplies a descriptor for every topology admitted to this contract. The common envelope names the backend and carries an opaque payload. ```rust struct TopologyDescriptor { @@ -125,7 +137,7 @@ struct TopologyDescriptor { `version` is the Isolation Backend interface version. Backend name and version match exactly; this contract does not negotiate compatibility ranges. The descriptor is transport-neutral. Provisioning supplies it to the supervisor before `attach`; how it is transported is topology-specific and outside this contract, and every transport preserves one property: workload-controlled input cannot select or modify the descriptor. -The opaque payload identifies, or gives the backend enough information to resolve, the exact driver-provisioned resource. It may also carry topology-specific endpoint or helper-role information; there are no common topology or role fields. +The opaque payload identifies an existing resource or gives the backend trusted prepared inputs with which to establish the exact resource during `attach`. It may also carry topology-specific endpoint or helper-role information; there are no common topology or role fields. Common verification requires: @@ -133,13 +145,13 @@ Common verification requires: - the descriptor's version is one the supervisor supports, and the resolved backend reports that same version; and - `SandboxContext` is constructed after the control plane assigns the resource to the admitted sandbox, using authenticated control-plane and trusted supervisor state. -The supervisor validates the descriptor's common fields and produces a `VerifiedTopologyDescriptor`, then resolves its `backend_name` and version without fallback. Verification does not imply that the opaque payload is valid; the selected backend validates it and atomically binds the provisioned resource to the trusted `SandboxContext` during `attach`. Any failure rejects the sandbox. +The supervisor validates the descriptor's common fields and produces a `VerifiedTopologyDescriptor`, then resolves its `backend_name` and version without fallback. Verification does not imply that the opaque payload is valid; the selected backend validates it and atomically establishes or locates the resource and binds it to the trusted `SandboxContext` during `attach`. Separated topologies also carry an opaque map of driver-owned immutable resource claims, such as a container ID, pod UID, or VM generation. Control mode presents those claims during authenticated attachment, and boundary mode compares them with its protected configuration before accepting the policy. Any failure rejects the sandbox. ### The lifecycle The contract does not prescribe enforcement mechanisms; it standardizes how the supervisor drives whichever backend a deployment admits. -A backend registers under a `backend_name` and version. The supervisor attaches to the admitted topology and drives the boundary through a fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. +A backend registers under a `backend_name` and version. The supervisor attaches to the admitted topology and drives the boundary through one fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. ```text attach topology + sandbox context -> Bound -> confirm -> Ready -> start_agent -> Running @@ -167,12 +179,23 @@ struct SandboxContext { #[async_trait] trait BoundBoundary: Send { fn network_mediation_source(&self) -> Arc; + fn dns_mediation_source(&self) -> Option>; + fn host_gateway_ip(&self) -> Option; async fn confirm( self: Box, ) -> Result, BackendError>; } +``` +`host_gateway_ip` is the backend's trusted host-side dial target for the +well-known host-gateway aliases. A backend returns it when the mediation +service runs outside the workload boundary and therefore cannot use the +boundary's resolver view; the supervisor preserves the original hostname for +policy, HTTP, and TLS while dialing the backend-provided address. `None` +leaves host-gateway discovery to the supervisor's local environment. + +```rust #[async_trait] trait ReadyBoundary: Send { async fn start_agent( @@ -190,7 +213,7 @@ trait RunningBoundary: Send + Sync { `AgentSpec` carries the complete admitted agent launch specification, including command, arguments, working directory, timeout, and interactive mode. -`SandboxContext` carries the admitted create-time policy. [RFC 0002](../0002-agent-driven-policy-management/README.md) defines how network-policy revisions are proposed and approved. Approved revisions reach the supervisor through the existing [`GetSandboxConfig`](../../proto/sandbox.proto) gateway-supervisor contract, described in the [gateway](../../architecture/gateway.md) and [sandbox](../../architecture/sandbox.md#policy-revision-acknowledgement) architecture. The supervisor makes approved network-policy revisions effective through network mediation. If an approved network-policy revision cannot be loaded, it never becomes effective; the configured rejection posture retains the last valid generation or denies network access until a valid generation is loaded. +`SandboxContext` carries the admitted launch-time policy. [RFC 0002](../0002-agent-driven-policy-management/README.md) defines how network-policy revisions are proposed and approved. Approved revisions reach the supervisor through the existing [`GetSandboxConfig`](../../proto/sandbox.proto) gateway-supervisor contract, described in the [gateway](../../architecture/gateway.md) and [sandbox](../../architecture/sandbox.md#policy-revision-acknowledgement) architecture. The supervisor makes approved network-policy revisions effective through network mediation. If an approved network-policy revision cannot be loaded, it never becomes effective; the configured rejection posture retains the last valid generation or denies network access until a valid generation is loaded. The states have normative meanings: @@ -200,7 +223,7 @@ The states have normative meanings: `confirm` is the pre-launch commit point. The supervisor calls it only after connecting the boundary's network-mediation source to network mediation. The backend confirms standing enforcement for the concrete boundary and may rely on a trusted provisioning-time or out-of-pod signal tied to that boundary's placement, but not on general placement health alone. -`attach` rejects a resource already bound to an active boundary. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `attach` or `confirm`, or the supervisor fails network-mediation initialization. +`attach` rejects a resource already bound to an active boundary or a conflicting launch generation. An idempotent retry resolves the same compatible inactive resource. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `attach` or `confirm`, or the supervisor fails network-mediation initialization. **Standing enforcement** is established independently of a workload process. **Launch-time controls** must be in force before a process executes its first untrusted instruction. Both `start_agent` and `BoundaryExec::exec` enforce this ordering and preserve the provisioned execution environment. @@ -257,11 +280,32 @@ trait NetworkMediationSource: Send + Sync { struct MediatedConnection { stream: BoundaryDuplexStream, binary_identity: Result, + destination: Option, +} + +#[async_trait] +trait DnsMediationSource: Send + Sync { + async fn accept(&self) -> Result; +} + +struct MediatedDnsQuery { + request: Vec, + transport: DnsTransport, + binary_identity: Result, + response: oneshot::Sender, BackendError>>, } ``` `NetworkMediationSource` supplies outbound connections from one boundary to supervisor-owned network mediation. The backend routes all workload egress through that source and authoritatively associates each connection with the boundary without relying solely on workload-provided data. Capture, transport, placement, and coordination are backend-private. +Explicit-proxy transports leave `destination` absent. Transparent transports +capture the original socket destination and supply it before the supervisor +consumes workload bytes. `DnsMediationSource` carries portless DNS exchanges to +the supervisor-owned policy DNS service. It is optional because explicit-proxy +topologies resolve destinations in the supervisor and do not expose workload +DNS. A backend that advertises transparent networking supplies both sources; +DNS or connection-source failure closes that boundary's egress. + Every topology may use the same supervisor-owned mediation libraries or services; the source does not require a backend-specific policy engine. Shared implementations isolate each boundary's state and enforcement. Failure or teardown of one boundary cannot weaken another. Network-mediation unavailability never enables direct egress. @@ -289,13 +333,13 @@ Binary identity is mandatory conformance: RFC 0002 makes it part of the outbound ### The supervisor sequence -The logical supervisor resolves `backend_name` and version through a trusted implementation registry. Adding a backend adds an implementation and registration, not branches in lifecycle, proxy, SSH, or session code. Delegated transport and coordination remain backend-private. +The logical supervisor resolves `backend_name` and version through a trusted implementation registry. A separated topology selects the reusable remote backend and supplies its standardized endpoint in the opaque descriptor. Adding a driver adds provisioning and endpoint construction in that driver's crate, not branches in lifecycle, proxy, SSH, session, or control-mode code. The supervisor runs the same sequence for every backend: -1. Obtain the `TopologyDescriptor` and trusted `SandboxContext`. +1. Obtain the trusted `TopologyDescriptor` and `SandboxContext` selected by admission. 2. Verify the descriptor and resolve its `backend_name` and version without fallback. -3. Call `attach` to obtain `Bound`. +3. Call `attach` to establish or locate the resource, bind it, and obtain `Bound`. 4. Connect the boundary's `NetworkMediationSource` to network mediation. 5. Call `confirm` to obtain `Ready`, then `start_agent` to obtain `Running`. 6. Use the returned runtime handles for agent wait, `exec`, and port forwarding while network mediation consumes outbound connections. @@ -312,12 +356,14 @@ enum BackendErrorKind { Invalid, Denied, Unavailable, Unsupported, Failed, Termi `Invalid` covers descriptor, version, and backend mismatches; `Denied` covers authenticated attachment rejection; `Unavailable` covers transient inability to serve an operation; `Unsupported` identifies an optional operation the selected backend does not implement; `Failed` covers other backend faults; and `Terminated` reports boundary or workload termination, or an operation against an inactive boundary. An error never advances the lifecycle or authorizes an operation, and backend selection never falls back. -A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per provisioned topology. If it does not return `Bound`, the topology is reclaimed rather than reused. +An `Unsupported` error variant reports that the selected backend does not implement an optional contract operation; it maps to the `Unavailable` kind for status purposes and never weakens a mandatory conformance requirement. + +A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per orchestration attempt. Attach-time establishment uses sandbox identity and launch generation as an idempotency key. If the operation does not return `Bound`, the compute driver reclaims the topology rather than reusing an ambiguous resource. Failures resolve as follows: -- an `attach` or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the driver to reclaim the topology; -- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the driver reclaims the topology; +- an `attach` or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the compute driver to reclaim the topology; +- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the compute driver reclaims the topology; - if `exec` or port-forward `connect` fails, the backend terminates any process or closes any connection created by that attempt while the boundary otherwise remains active; - after `Running`, supervisor or enforcement loss follows invariant 6; when enforcement loss ends the agent, `BoundaryProcess::wait` fails with `BackendErrorKind::Terminated` where process-exit observation survives; - network-mediation errors yield no authorized connection and do not by themselves end `Running`; and @@ -331,11 +377,12 @@ The contract fixes the roles; a topology fixes their placement. Components may b ## Implementation plan -This RFC defines the contract; implementation lands in three phases: +This RFC defines the contract; implementation lands in four phases: 1. **Contract.** Add the common types, descriptor handling, registry, and explicit backend selection from deployment configuration. -2. **Co-located backend.** Implement the co-located backend behind a deployment flag and route agent launch, egress interception, the network-mediation source, SSH, `exec`, and forwarding through it without changing behavior. -3. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus descriptor verification, lifecycle ordering, runtime operations, and failure semantics. Make the co-located backend the default after parity validation. Parity covers the agent, binary identity, SSH, `exec`, and forwarding paths; enablement also closes the in-pod egress gaps pinned in [codebase-grounding.md](./codebase-grounding.md), which parity alone would preserve. +2. **Shared supervisor modes.** Add the versioned boundary protocol, reusable remote backend, and the `control` and `boundary` entrypoints to `openshell-sandbox`. +3. **Driver adoption.** Have VM, Docker, and Kubernetes provision protected boundary configs and topology descriptors in their existing driver crates. No adoption may require a shared-supervisor change; that constraint is the abstraction test. +4. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus descriptor verification, lifecycle ordering, runtime operations, compute-driver-owned cleanup, and failure semantics. Existing placements remain outside this contract until their backend is implemented and admitted; they do not claim conformance. Delegated backends remain separate design and implementation work. @@ -343,7 +390,8 @@ Existing placements remain outside this contract until their backend is implemen | Risk | Mitigation | |---|---| -| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep the responsibility boundary explicit: the compute driver owns, provisions, and deprovisions the topology; the backend binds and operates the active boundary. The same component may implement both roles. | +| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep placement, preparation, durable deletion, reconciliation, and endpoint selection in the compute driver; keep enforcement sequencing in control mode and boundary-local observation in boundary mode. Generic supervisor code never imports a concrete driver. | +| Attach-time establishment could leave a partial resource after supervisor failure. | Make establishment idempotent by sandbox launch generation, label resources for compute-driver reconciliation, and do not start untrusted code before `attach` and `confirm` complete. | | Contract conformance could be mistaken for equivalent isolation across topologies. | Treat conformance as behavioral, not as a security-strength rating. Document and validate each topology's actual containment and reject policy it cannot enforce. | | Shared backend or network-mediation components concentrate privilege and failure impact. | Isolate state, connection attribution, enforcement, and control authority per boundary. Failure of one boundary must not weaken another or enable direct egress. | | The mandatory contract may exclude otherwise useful but incomplete backends. | Keep the network-mediation source, binary identity, process control, `exec`, and port forwarding mandatory. An incomplete backend does not claim conformance or silently degrade. | @@ -358,17 +406,17 @@ OpenShell could keep the current in-pod design and add topology-specific supervi Doing nothing avoids a new interface, but retains privileged boundary construction beside the workload. Implementing each delegated topology as a one-off supervisor change moves that privilege for one placement but accretes topology-specific supervisor behavior. The proposed contract instead keeps one supervisor lifecycle while allowing the topology to change. -### Extend the compute-driver contract +### Require every resource to exist before attach -The compute driver could own both provisioning and active-boundary operation. +The compute driver could always create the concrete resource before the supervisor calls `attach`. -This is natural for topologies such as MXC, and the same component may implement both responsibilities. The interfaces remain distinct because they serve different callers and lifecycles: the gateway uses the compute driver to provision and deprovision resources, while the supervisor uses the Isolation Backend to operate an active boundary. Combining them would couple runtime policy, identity, network mediation, and process operations to the gateway-facing driver API. +This remains natural for controller-driven systems such as Kubernetes. Requiring it everywhere prevents a local backend from establishing host listeners and enforcement state before a runtime creates the workload. Allowing a trusted topology descriptor to carry prepared inputs preserves the single `attach` contract while allowing security-sensitive establishment to remain atomic with binding. The compute driver still owns deletion and reconciliation. -### Start with a remote backend service +### Give each driver its own remote control service -The contract could be expressed as a gRPC service or plugin ABI rather than an in-process Rust contract. [RFC 0001](../0001-core-architecture/README.md) chose gRPC for its gateway-facing drivers, so the question applies here. +Each separated driver could define a private gRPC service, guest agent, or runtime-specific plugin ABI behind its `IsolationBackend` implementation. -The callers differ. A gateway driver is a control-plane peer with its own release cycle, while the Isolation Backend is driven by the supervisor that operates the boundary, and the co-located topology needs no transport at all. Starting in-process serves that case directly and lets delegated implementations carry their own transport behind the same interface. A transport-bearing surface is not precluded: it is versioned contract surface, added when a concrete delegated backend requires it. +That would hide transport differences from the Rust trait, but it would duplicate lifecycle, authentication, process streaming, signaling, forwarding, and binary-identity semantics across Docker, Kubernetes, and VM implementations. The proposed versioned boundary protocol standardizes those semantics once. Drivers still choose and provision Unix socket, authenticated TCP, vsock, or adapter transport and bind their own immutable resource claims. ### Standardize topology and capabilities @@ -381,10 +429,11 @@ That would make known deployments explicit, but it would also encode current top - **Driver-backed subsystems (CRI/CNI/CSI).** Kubernetes factors runtime, networking, and storage into pluggable driver contracts so the orchestrator drives one interface while implementations vary. RFC 0001 describes OpenShell's other subsystems the same way; this RFC specifies the one it left open: isolation. - **Istio privilege placement.** Init-sidecar and node-agent modes demonstrate that network setup can move without changing the policy data path. OpenShell keeps its identity-aware proxy. - **CRI exec/attach/port-forward.** `exec` and `connect` follow CRI's `Exec` and `PortForward` shape; lifecycle and network mediation remain OpenShell-specific. +- **[OCI seccomp listener handoff](https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#seccomp).** The runtime specification lets a runtime send a seccomp notification FD and process state to a host Unix listener. It demonstrates why some local enforcement must exist before workload creation and informs attach-time boundary establishment. ## Open questions -None. +None for this revision. New common fields require evidence from a concrete backend and a protocol-version change when compatibility cannot be preserved. ## Appendix: codebase grounding diff --git a/rfc/0012-isolation-backend/topology-matrix.md b/rfc/0012-isolation-backend/topology-matrix.md index 8dc14d84ba..bd5a8167e0 100644 --- a/rfc/0012-isolation-backend/topology-matrix.md +++ b/rfc/0012-isolation-backend/topology-matrix.md @@ -6,18 +6,24 @@ kernel; it does not select a deployment or establish conformance. ## Representative placements -| Pattern | Logical supervisor and network-mediation placement | Backend placement | Workload-kernel relationship | Topology status | +| Pattern | Control and network-mediation placement | Boundary-mode placement | Workload-kernel relationship | Topology status | |---|---|---|---|---| -| **Co-located/in-pod** | With the workload | In the supervisor process | Trusted components share the workload's host, guest, or application kernel, depending on the runtime | Placement implemented (original topology) | -| **Same-pod composite** | Spans the workload-local supervisor process and, when used, a network-mediation sidecar | In the workload-local supervisor process | Components share the workload's kernel | Placement implemented (#2076) | -| **Delegated backend components** | With the workload and any delegated mediation component | A node or remote helper establishes some controls behind a workload-local backend | Depends on which trusted components remain with the workload | Placement proposed (#2606) | -| **Driver-hosted/shared service** | With the compute driver or another trusted service; no in-sandbox supervisor process is required | May be co-located with the logical supervisor; one host may operate many isolated boundaries | Depends on the workload runtime | Placement proposed | +| **Co-located/in-pod** | With the workload; the legacy in-process backend may omit boundary mode | Same process when used | Trusted components share the workload's host kernel | Placement implemented (original topology) | +| **Kubernetes proxy pod** | Trusted proxy pod | Trusted sidecar or companion sharing workload PID and network observation | Shared node kernel; pod security boundaries separate control from workload | Prototype in #3144 | +| **Docker** | Gateway host | Trusted container entrypoint sharing the workload container's PID and network namespaces | Shared host kernel | Prototype in #2965 | +| **MicroVM** | Gateway host | Guest PID 1 | Boundary mode shares the guest kernel; control is kernel-separated | Prototype in #2945 | + +The Kubernetes proxy-pod topology uses the same authenticated boundary protocol +as Docker and VM. Kubernetes-specific code provisions pod placement, the +private transport, and immutable pod/Sandbox resource claims. It still requires +mediated DNS and attributed connections; pod readiness or the existence of a +`NetworkPolicy` object alone does not confirm enforcement. ## Durable rules - Every active boundary has one verified descriptor, one trusted - `SandboxContext`, and at most one logical supervisor, which may span multiple - coupled processes. + `SandboxContext`, one control role, and at most one boundary role. Those + processes form one logical supervisor. - Physical processes and listeners may be shared, but lifecycle state, policy, binary identity, enforcement, and cleanup remain isolated per boundary. - Moving a privileged component does not itself provide kernel separation. From 19f229d52e6d86e6eb7a113b3f7a89bb8c2779eb Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 2 Sep 2026 21:50:12 -0700 Subject: [PATCH 02/37] fix(isolation): attach control to boundary-owned agent Signed-off-by: Drew Newberry --- .../src/boundary_protocol.rs | 8 + .../src/contract.rs | 19 ++ .../src/remote.rs | 76 ++++++- .../openshell-sandbox/src/boundary_server.rs | 133 +++++++++++- crates/openshell-sandbox/src/lib.rs | 4 +- .../src/delegated.rs | 32 ++- .../src/main_session.rs | 189 +++++++++++++++++- .../openshell-supervisor-process/src/ssh.rs | 20 +- 8 files changed, 463 insertions(+), 18 deletions(-) diff --git a/crates/openshell-isolation-interface/src/boundary_protocol.rs b/crates/openshell-isolation-interface/src/boundary_protocol.rs index efd233a5ab..0f26c1056f 100644 --- a/crates/openshell-isolation-interface/src/boundary_protocol.rs +++ b/crates/openshell-isolation-interface/src/boundary_protocol.rs @@ -286,6 +286,9 @@ pub enum Request { ca_bundle: Option>, provider_env: std::collections::HashMap, }, + AttachProcess { + process_id: String, + }, Wait { process_id: String, }, @@ -350,6 +353,10 @@ impl fmt::Debug for Request { .debug_struct("Wait") .field("process_id", process_id) .finish(), + Self::AttachProcess { process_id } => formatter + .debug_struct("AttachProcess") + .field("process_id", process_id) + .finish(), Self::Signal { process_id, signal } => formatter .debug_struct("Signal") .field("process_id", process_id) @@ -397,6 +404,7 @@ pub enum Response { Attached, Confirmed, Started { process_id: String }, + ProcessAttached { terminal: bool }, Exited { status: ExitStatusWire }, Signaled, Terminated, diff --git a/crates/openshell-isolation-interface/src/contract.rs b/crates/openshell-isolation-interface/src/contract.rs index 89fcf7ce48..52c0e60326 100644 --- a/crates/openshell-isolation-interface/src/contract.rs +++ b/crates/openshell-isolation-interface/src/contract.rs @@ -427,6 +427,13 @@ pub enum BoundarySignal { /// however many times it is called; a local PID is never the process handle. #[async_trait] pub trait BoundaryProcess: Send + Sync { + /// Attach to the admitted process's retained standard I/O. The boundary + /// remains the process owner and may permit only one control attachment. + async fn attach(&self) -> Result { + Err(BackendError::Unsupported( + "process attachment is not supported".to_string(), + )) + } /// Await terminal status (stable across repeated calls). async fn wait(&self) -> Result; /// Deliver a signal to the process or its group. @@ -440,6 +447,18 @@ pub type BoundaryInput = Box; /// A boxed async reader from a boundary process's stdout or stderr. pub type BoundaryOutput = Box; +/// A control-side attachment to the admitted process's retained I/O. +pub struct ProcessAttachment { + /// Stdin writer. + pub stdin: BoundaryInput, + /// Stdout reader, or the PTY-merged output stream. + pub stdout: BoundaryOutput, + /// Stderr reader, distinct from stdout for non-PTY processes. + pub stderr: Option, + /// PTY control, present when the admitted process owns a terminal. + pub terminal: Option>, +} + /// A PTY attached to an exec session. #[async_trait] pub trait BoundaryTerminal: Send + Sync { diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index 0ea35aafcb..6c0528080f 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -17,8 +17,8 @@ use crate::contract::{ BackendError, BoundBoundary, BoundaryDuplexStream, BoundaryExec, BoundaryExitStatus, BoundaryInput, BoundaryOutput, BoundaryPortForward, BoundaryProcess, BoundarySignal, BoundaryTerminal, ExecSession, ExecSpec, INTERFACE_VERSION, IsolationBackend, LoopbackTarget, - MediatedConnection, NetworkMediationSource, ReadyBoundary, RunningBoundary, SandboxContext, - VerifiedTopologyDescriptor, + MediatedConnection, NetworkMediationSource, ProcessAttachment, ReadyBoundary, RunningBoundary, + SandboxContext, VerifiedTopologyDescriptor, }; use async_trait::async_trait; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -296,6 +296,10 @@ struct RemoteProcess { #[async_trait] impl BoundaryProcess for RemoteProcess { + async fn attach(&self) -> Result { + open_process_attachment(self.client.clone(), self.process_id.clone()).await + } + async fn wait(&self) -> Result { let response = self .client @@ -340,6 +344,74 @@ impl BoundaryProcess for RemoteProcess { } } +async fn open_process_attachment( + client: Arc, + process_id: String, +) -> Result { + let (stream, response) = client + .call_stream(Request::AttachProcess { + process_id: process_id.clone(), + }) + .await?; + let Response::ProcessAttached { + terminal: has_terminal, + } = response + else { + return Err(unexpected_response("process_attached", &response)); + }; + let (network_reader, network_writer) = tokio::io::split(stream); + let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); + let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); + let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); + tokio::spawn(pump_exec_input(stdin_pump, network_writer)); + tokio::spawn(pump_process_responses( + network_reader, + stdout_pump, + stderr_pump, + )); + let terminal: Option> = if has_terminal { + let terminal: Arc = Arc::new(RemoteTerminal { client, process_id }); + Some(terminal) + } else { + None + }; + let stderr: Option = if has_terminal { + None + } else { + let stderr: BoundaryOutput = Box::new(stderr); + Some(stderr) + }; + Ok(ProcessAttachment { + stdin: Box::new(stdin), + stdout: Box::new(stdout), + stderr, + terminal, + }) +} + +async fn pump_process_responses( + mut network: tokio::io::ReadHalf, + mut stdout: tokio::io::DuplexStream, + mut stderr: tokio::io::DuplexStream, +) { + loop { + match read_stream_frame(&mut network).await { + Ok(Some((STREAM_STDOUT, payload))) => { + if stdout.write_all(&payload).await.is_err() { + return; + } + } + Ok(Some((STREAM_STDERR, payload))) => { + if stderr.write_all(&payload).await.is_err() { + return; + } + } + Ok(Some((STREAM_EXIT, _)) | None) | Err(_) => return, + Ok(Some((_channel, _))) => return, + } + } +} + struct RemoteExec { client: Arc, } diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 6574f4fc06..eea84891c3 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -35,6 +35,7 @@ mod linux { use openshell_supervisor_process::boundary_io::BoundaryRuntimeState; use openshell_supervisor_process::delegated::{AgentSignaler, spawn_workload}; use openshell_supervisor_process::identity::{DriverIdentity, resolve_process_identity}; + use openshell_supervisor_process::main_session::{MainOutput, MainSession}; use openshell_supervisor_process::netns::{ NetworkNamespace, create_conformant_netns_for_proxy, }; @@ -259,6 +260,30 @@ mod linux { .map_err(|error| format!("write exec start response: {error}"))?; return runtime.stream_exec(stream, &process_id, session); } + Request::AttachProcess { process_id } => { + let (session, terminal) = match runtime.attach_process(&process_id) { + Ok(attachment) => attachment, + Err(response) => { + return write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response, + }, + ) + .map_err(|error| format!("write process attachment error: {error}")); + } + }; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::ProcessAttached { terminal }, + }, + ) + .map_err(|error| format!("write process attachment response: {error}"))?; + return runtime.stream_process(stream, session); + } Request::PortForward { host, port } => { let target = LoopbackTarget::new(host, port) .map_err(|error| format!("validate port-forward target: {error}"))?; @@ -507,8 +532,11 @@ mod linux { process_id, cols, rows, - } => self.resize_exec(&process_id, cols, rows), - Request::Exec { .. } | Request::PortForward { .. } | Request::AcceptNetwork => { + } => self.resize_process(&process_id, cols, rows), + Request::Exec { .. } + | Request::AttachProcess { .. } + | Request::PortForward { .. } + | Request::AcceptNetwork => { guest_error("invalid", "streaming request used on control path") } } @@ -560,7 +588,20 @@ mod linux { } } - fn resize_exec(&self, process_id: &str, cols: u16, rows: u16) -> Response { + fn resize_process(&self, process_id: &str, cols: u16, rows: u16) -> Response { + if let Ok(process) = self.running_process(process_id) { + let session = process.main_session(); + if !session.terminal() { + return guest_error("invalid", "agent process has no terminal"); + } + self.process_runtime.block_on(session.resize( + u32::from(cols), + u32::from(rows), + 0, + 0, + )); + return Response::Resized; + } let terminal = lock(&self.exec_handles) .get(process_id) .and_then(|handle| handle.terminal.clone()); @@ -636,6 +677,29 @@ mod linux { bridged } + fn attach_process(&self, process_id: &str) -> Result<(Arc, bool), Response> { + let process = self.running_process(process_id)?; + if process.attached.swap(true, Ordering::AcqRel) { + return Err(guest_error( + "denied", + "main process already has a control attachment", + )); + } + let session = process.main_session(); + Ok((session.clone(), session.terminal())) + } + + fn stream_process( + &self, + stream: ControlStream, + session: Arc, + ) -> Result<(), String> { + self.process_runtime.block_on(async move { + let stream = stream.into_tokio()?; + bridge_main_stream(stream, session).await + }) + } + fn attach(&self, policy: SandboxPolicyWire) -> Response { let mut state = lock(&self.state); match &*state { @@ -867,6 +931,8 @@ mod linux { port_forward: Arc, network_listener: Option>, identity_resolver: ProcfsIdentityResolver, + main_session: Arc, + attached: AtomicBool, _netns: Option>, } @@ -930,6 +996,7 @@ mod linux { let signaler = spawned.signaler(); let boundary_exec = spawned.boundary_exec(); let port_forward = spawned.port_forward(); + let main_session = spawned.main_session(); let network_listener = prepared.network_listener.clone(); let exit = Arc::new((Mutex::new(None), Condvar::new())); let reaper_exit = exit.clone(); @@ -951,6 +1018,8 @@ mod linux { port_forward, network_listener, identity_resolver: ProcfsIdentityResolver { entrypoint_pid }, + main_session, + attached: AtomicBool::new(false), _netns: prepared.netns, }) } @@ -1003,6 +1072,64 @@ mod linux { fn identity_resolver(&self) -> ProcfsIdentityResolver { self.identity_resolver.clone() } + + fn main_session(&self) -> Arc { + self.main_session.clone() + } + } + + async fn bridge_main_stream( + stream: openshell_isolation_interface::contract::BoundaryDuplexStream, + session: Arc, + ) -> Result<(), String> { + let (mut reader, writer) = tokio::io::split(stream); + let writer = Arc::new(tokio::sync::Mutex::new(writer)); + let (owner, input) = session.acquire_input().map_err(str::to_string)?; + let mut output = session.subscribe(); + let input_task = tokio::spawn(async move { + while let Some((channel, payload)) = read_stream_frame(&mut reader).await? { + match channel { + STREAM_STDIN => input.send(payload).await.map_err(|_| { + io::Error::new(io::ErrorKind::BrokenPipe, "main process stdin closed") + })?, + STREAM_STDIN_CLOSED => break, + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected host-to-boundary main stream channel", + )); + } + } + } + Ok::<(), io::Error>(()) + }); + let result = loop { + let (channel, payload) = match output.recv().await { + Ok(MainOutput::Stdout(payload)) => (STREAM_STDOUT, payload.to_vec()), + Ok(MainOutput::Stderr(payload)) => (STREAM_STDERR, payload.to_vec()), + Ok(MainOutput::Exit(code)) => { + let status = serde_json::to_vec(&ExitStatusWire::Exited(code)) + .map_err(|error| format!("encode main process exit: {error}"))?; + break write_stream_frame(&mut *writer.lock().await, STREAM_EXIT, &status) + .await + .map_err(|error| format!("write main process exit: {error}")); + } + Err(error) => { + break Err(format!( + "main process attachment lagged by {} chunks", + error.skipped + )); + } + }; + if let Err(error) = + write_stream_frame(&mut *writer.lock().await, channel, &payload).await + { + break Err(format!("write main process output: {error}")); + } + }; + input_task.abort(); + session.release_input(owner); + result } fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 99aa97c954..c0c1e89d59 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -961,6 +961,7 @@ pub async fn run_sandbox( .await .map_err(|error| miette::miette!(error.to_string()))?; info!(backend = %backend_name, "Isolation boundary agent started"); + let agent = running.agent(); let boundary_access = openshell_supervisor_process::delegated::start_boundary_access( sandbox_id.as_deref(), openshell_endpoint.as_deref(), @@ -972,11 +973,11 @@ pub async fn run_sandbox( process_enforcement_mode, running.exec(), running.port_forward(), + agent.clone(), ) .await?; info!(backend = %backend_name, "Control-mode access plane started"); let instance_id = boundary_access.instance_id().to_string(); - let agent = running.agent(); let wait_agent = agent.clone(); let wait = async move { wait_agent @@ -1010,6 +1011,7 @@ pub async fn run_sandbox( ) .await; } + boundary_access.finish_main(exit_code).await; drop(running); drop(boundary_access); exit_code diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 9e5defd20b..8dcceead64 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -16,7 +16,7 @@ use miette::{IntoDiagnostic as _, Result}; use openshell_core::policy::{NetworkMode, SandboxPolicy}; use openshell_core::proposals::AgentProposals; use openshell_core::provider_credentials::ProviderCredentialState; -use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward}; +use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward, BoundaryProcess}; use openshell_ocsf::{ ActionId, ActivityId, AppLifecycleBuilder, DispositionId, LaunchTypeId, Process as OcsfProcess, ProcessActivityBuilder, SeverityId, StatusId, ocsf_emit, @@ -39,6 +39,7 @@ pub struct BoundaryAccess { terminating: Arc, ssh_task: Option>, session_task: Option>, + main_session: Option>, } impl BoundaryAccess { @@ -47,6 +48,17 @@ impl BoundaryAccess { pub fn instance_id(&self) -> &str { &self.instance_id } + + /// Complete the control-side canonical-process attachment after the + /// boundary reports terminal status. + pub async fn finish_main(&self, exit_code: i32) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; + let _ = main_session.finish(exit_code, false).await; + main_session.mark_terminal_reported(); + main_session.wait_for_terminal_attachments().await; + } } impl Drop for BoundaryAccess { @@ -73,6 +85,7 @@ pub async fn start_boundary_access( enforcement_mode: ProcessEnforcementMode, boundary_exec: Arc, port_forward: Arc, + agent: Arc, ) -> Result { let instance_id = uuid::Uuid::new_v4().to_string(); let terminating = Arc::new(AtomicBool::new(false)); @@ -82,12 +95,20 @@ pub async fn start_boundary_access( terminating, ssh_task: None, session_task: None, + main_session: None, }); }; + let attachment = agent + .attach() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + let main_session = crate::main_session::MainSession::from_boundary(attachment, agent); + let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); let listen_path = ssh_socket_path.clone(); let ssh_port_forward = port_forward.clone(); + let ssh_main_session = main_session.clone(); let ssh_task = tokio::spawn(async move { if let Err(error) = crate::ssh::run_ssh_server( listen_path, @@ -97,7 +118,7 @@ pub async fn start_boundary_access( shared_ssh_socket, ssh_port_forward, boundary_exec, - None, + Some(ssh_main_session), ) .await { @@ -150,6 +171,7 @@ pub async fn start_boundary_access( terminating, ssh_task: Some(ssh_task), session_task, + main_session: Some(main_session), }) } @@ -376,6 +398,12 @@ impl SpawnedAgent { self.port_forward.clone() } + /// Retained canonical-process I/O owned by the boundary. + #[must_use] + pub fn main_session(&self) -> Arc { + self.main_session.clone() + } + /// Wait for exit, enforcing the admitted wall-clock timeout. pub async fn wait(&mut self) -> Result { let signaler = self.signaler(); diff --git a/crates/openshell-supervisor-process/src/main_session.rs b/crates/openshell-supervisor-process/src/main_session.rs index 00dd2ea53c..e91bf9d0f2 100644 --- a/crates/openshell-supervisor-process/src/main_session.rs +++ b/crates/openshell-supervisor-process/src/main_session.rs @@ -17,6 +17,10 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::Notify; use tokio::sync::watch; +use openshell_isolation_interface::contract::{ + BoundaryProcess, BoundarySignal, BoundaryTerminal, ProcessAttachment, +}; + use crate::process::ProcessIo; const OUTPUT_BUFFER_BYTES: usize = 1024 * 1024; @@ -186,6 +190,8 @@ pub struct MainSession { input_owner: Mutex>, next_owner: AtomicU64, pty_master: Option>, + boundary_process: Option>, + boundary_terminal: Option>, readers_remaining: AtomicUsize, readers_done: Notify, finished: std::sync::atomic::AtomicBool, @@ -205,6 +211,8 @@ impl MainSession { input_owner: Mutex::new(None), next_owner: AtomicU64::new(1), pty_master: None, + boundary_process: None, + boundary_terminal: None, readers_remaining: AtomicUsize::new(0), readers_done: Notify::new(), finished: std::sync::atomic::AtomicBool::new(false), @@ -256,6 +264,8 @@ impl MainSession { input_owner: Mutex::new(None), next_owner: AtomicU64::new(1), pty_master, + boundary_process: None, + boundary_terminal: None, readers_remaining: AtomicUsize::new(if terminal { 1 } else { 2 }), readers_done: Notify::new(), finished: std::sync::atomic::AtomicBool::new(false), @@ -270,6 +280,81 @@ impl MainSession { session } + /// Build the control-side multiplexer around a boundary-owned admitted + /// process. Process lifecycle and PTY operations remain delegated to the + /// boundary process handle. + #[must_use] + pub fn from_boundary( + attachment: ProcessAttachment, + process: Arc, + ) -> Arc { + let ProcessAttachment { + stdin, + stdout, + stderr, + terminal, + } = attachment; + let terminal_mode = terminal.is_some(); + let (input, mut input_rx) = tokio::sync::mpsc::channel::>(64); + let session = Arc::new(Self { + pid: 0, + terminal: terminal_mode, + input, + output: OutputLog::new(), + input_owner: Mutex::new(None), + next_owner: AtomicU64::new(1), + pty_master: None, + boundary_process: Some(process), + boundary_terminal: terminal, + readers_remaining: AtomicUsize::new(if terminal_mode { 1 } else { 2 }), + readers_done: Notify::new(), + finished: std::sync::atomic::AtomicBool::new(false), + terminal_attachments: Mutex::new(TerminalAttachmentState { + active: 0, + process_finished: false, + expectation: AttachmentExpectation::None, + }), + terminal_attachments_done: Notify::new(), + }); + let stdout_session = Arc::clone(&session); + tokio::spawn(async move { + let mut stdout = stdout; + let mut buffer = [0u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stdout_session + .publish(MainOutput::Stdout(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stdout_session.reader_finished(); + }); + if let Some(mut stderr) = stderr { + let stderr_session = Arc::clone(&session); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stderr_session + .publish(MainOutput::Stderr(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stderr_session.reader_finished(); + }); + } + tokio::spawn(async move { + let mut stdin = stdin; + while let Some(data) = input_rx.recv().await { + if stdin.write_all(&data).await.is_err() { + break; + } + let _ = stdin.flush().await; + } + }); + session + } + fn start_io( this: &Arc, io: ProcessIo, @@ -499,7 +584,16 @@ impl MainSession { } } - pub fn resize(&self, columns: u32, rows: u32, pixel_width: u32, pixel_height: u32) { + pub async fn resize(&self, columns: u32, rows: u32, pixel_width: u32, pixel_height: u32) { + if let Some(terminal) = self.boundary_terminal.as_ref() { + let _ = terminal + .resize( + u16::try_from(columns.max(1)).unwrap_or(u16::MAX), + u16::try_from(rows.max(1)).unwrap_or(u16::MAX), + ) + .await; + return; + } let Some(master) = self.pty_master.as_ref() else { return; }; @@ -515,9 +609,23 @@ impl MainSession { } } - pub fn signal_group(&self, signal: nix::sys::signal::Signal) -> Result<(), nix::errno::Errno> { + pub async fn signal_group(&self, signal: nix::sys::signal::Signal) -> Result<(), String> { + if let Some(process) = self.boundary_process.as_ref() { + let signal = match signal { + nix::sys::signal::Signal::SIGHUP => BoundarySignal::Hup, + nix::sys::signal::Signal::SIGINT => BoundarySignal::Int, + nix::sys::signal::Signal::SIGKILL => BoundarySignal::Kill, + nix::sys::signal::Signal::SIGTERM => BoundarySignal::Term, + other => return Err(format!("boundary signal {other:?} is unsupported")), + }; + return process + .signal(signal) + .await + .map_err(|error| error.to_string()); + } let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); nix::sys::signal::kill(nix::unistd::Pid::from_raw(-pid), signal) + .map_err(|error| error.to_string()) } #[must_use] @@ -544,6 +652,83 @@ fn set_nonblocking(file: &std::fs::File) -> Result<(), nix::errno::Errno> { #[cfg(test)] mod tests { use super::*; + use openshell_isolation_interface::contract::{ + BackendError, BoundaryExitStatus, BoundaryInput, BoundaryOutput, + }; + + struct TestBoundaryProcess { + signals: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryProcess for TestBoundaryProcess { + async fn wait(&self) -> Result { + Ok(BoundaryExitStatus::Exited(0)) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + self.signals.lock().unwrap().push(signal); + Ok(()) + } + + async fn terminate(&self) -> Result<(), BackendError> { + Ok(()) + } + } + + struct TestBoundaryTerminal { + size: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryTerminal for TestBoundaryTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + *self.size.lock().unwrap() = Some((cols, rows)); + Ok(()) + } + } + + #[tokio::test] + async fn boundary_attachment_drives_main_io_signal_and_terminal() { + let (stdin, mut stdin_peer) = tokio::io::duplex(1024); + let (stdout, mut stdout_peer) = tokio::io::duplex(1024); + let process = Arc::new(TestBoundaryProcess { + signals: Mutex::new(Vec::new()), + }); + let terminal = Arc::new(TestBoundaryTerminal { + size: Mutex::new(None), + }); + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let attachment = ProcessAttachment { + stdin, + stdout, + stderr: None, + terminal: Some(terminal.clone()), + }; + let session = MainSession::from_boundary(attachment, process.clone()); + let mut output = session.subscribe(); + + stdout_peer.write_all(b"ready\n").await.unwrap(); + assert!(matches!( + output.recv().await.unwrap(), + MainOutput::Stdout(data) if data == b"ready\n"[..] + )); + + let (_owner, input) = session.acquire_input().unwrap(); + input.send(b"hello\n".to_vec()).await.unwrap(); + let mut received = [0_u8; 6]; + stdin_peer.read_exact(&mut received).await.unwrap(); + assert_eq!(&received, b"hello\n"); + + session.resize(120, 40, 0, 0).await; + assert_eq!(*terminal.size.lock().unwrap(), Some((120, 40))); + session + .signal_group(nix::sys::signal::Signal::SIGINT) + .await + .unwrap(); + assert_eq!(*process.signals.lock().unwrap(), vec![BoundarySignal::Int]); + } #[test] fn input_lease_has_one_owner_and_can_be_reacquired() { diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 0b4101160f..b312d7c7b1 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -493,7 +493,9 @@ impl russh::server::Handler for SshHandler { }; if state.main_attached { if let Some(main_session) = self.main_session.as_ref() { - main_session.resize(col_width, row_height, pixel_width, pixel_height); + main_session + .resize(col_width, row_height, pixel_width, pixel_height) + .await; } } else if let Some(terminal) = state.terminal.as_ref() && let Err(e) = terminal @@ -562,12 +564,14 @@ impl russh::server::Handler for SshHandler { .expect("main channel existence checked above"); state.main_attached = true; if let Some(pty) = state.pty_request.take() { - main_session.resize( - pty.col_width, - pty.row_height, - pty.pixel_width, - pty.pixel_height, - ); + main_session + .resize( + pty.col_width, + pty.row_height, + pty.pixel_width, + pty.pixel_height, + ) + .await; } let (input, warning) = if state.main_read_only { (None, None) @@ -757,7 +761,7 @@ impl russh::server::Handler for SshHandler { _ => None, }; if let (Some(signal), Some(main_session)) = (signal, self.main_session.as_ref()) - && let Err(error) = main_session.signal_group(signal) + && let Err(error) = main_session.signal_group(signal).await { warn!(%error, ?signal, "failed to signal canonical main process group"); } From b9a9455ac692b3b37ddc669e3b8af097a25be3d9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 2 Sep 2026 21:57:34 -0700 Subject: [PATCH 03/37] fix(supervisor): allow late main process attachment Signed-off-by: Drew Newberry --- .../openshell-supervisor-process/src/ssh.rs | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index b312d7c7b1..2db6e7c66a 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -550,11 +550,7 @@ impl russh::server::Handler for SshHandler { session.channel_failure(channel)?; return Ok(()); }; - if !self.channels.contains_key(&channel) || main_session.finished() { - session.channel_failure(channel)?; - return Ok(()); - } - if main_session.begin_terminal_attachment().is_err() { + if !begin_main_attachment(&main_session, self.channels.contains_key(&channel)) { session.channel_failure(channel)?; return Ok(()); } @@ -972,6 +968,10 @@ impl SshHandler { } } +fn begin_main_attachment(main_session: &MainSession, channel_exists: bool) -> bool { + channel_exists && main_session.begin_terminal_attachment().is_ok() +} + async fn send_main_output(handle: &Handle, channel: ChannelId, event: MainOutput) -> bool { match event { MainOutput::Stdout(data) => handle.data(channel, data).await.is_ok(), @@ -1760,6 +1760,21 @@ mod tests { use super::*; use std::process::Stdio; + #[tokio::test] + async fn main_attachment_accepts_declared_session_after_process_exit() { + let main_session = MainSession::inert(); + assert!(main_session.finish(23, true).await); + assert!(main_session.finished()); + + assert!(begin_main_attachment(&main_session, true)); + let mut output = main_session.subscribe(); + assert!(matches!( + output.recv().await.expect("retained terminal status"), + MainOutput::Exit(23) + )); + main_session.end_terminal_attachment(); + } + #[cfg(unix)] fn file_mode(path: &Path) -> u32 { use std::os::unix::fs::PermissionsExt; From b5800b46f11019775721e5b2e37dadb894079add Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 2 Sep 2026 22:00:45 -0700 Subject: [PATCH 04/37] fix(vm): build complete guest supervisor bundle Signed-off-by: Drew Newberry --- e2e/rust/e2e-vm.sh | 8 ++-- tasks/scripts/vm/build-supervisor-bundle.sh | 46 +++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 96da2a879f..3805a2d092 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -88,10 +88,10 @@ if [ -z "${OPENSHELL_VM_DRIVER_BIN:-}" ]; then mise run vm:setup fi - if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then - echo "==> Building bundled VM supervisor (mise run vm:supervisor)" - mise run vm:supervisor - fi + # Always rebuild the guest bundle so an e2e run cannot silently exercise a + # stale boundary binary after supervisor or isolation-interface changes. + echo "==> Building bundled VM supervisor (mise run vm:supervisor)" + mise run vm:supervisor export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR}}" else diff --git a/tasks/scripts/vm/build-supervisor-bundle.sh b/tasks/scripts/vm/build-supervisor-bundle.sh index 0085c0619d..a31e999603 100755 --- a/tasks/scripts/vm/build-supervisor-bundle.sh +++ b/tasks/scripts/vm/build-supervisor-bundle.sh @@ -60,6 +60,7 @@ esac SUPERVISOR_BIN="${ROOT}/target/${RUST_TARGET}/release/openshell-sandbox" SUPERVISOR_OUTPUT="${OUTPUT_DIR}/openshell-sandbox.zst" +SUPERVISOR_RUNTIME_OUTPUT="${OUTPUT_DIR}/openshell-runtime.tar.zst" echo "==> Building openshell-sandbox supervisor bundle" echo " Guest arch: ${GUEST_ARCH}" @@ -123,6 +124,51 @@ fi zstd -19 -T0 -f "${SUPERVISOR_BIN}" -o "${SUPERVISOR_OUTPUT}" +case "${GUEST_ARCH}" in + aarch64|arm64) DOCKER_ARCH="arm64" ;; + x86_64|amd64) DOCKER_ARCH="amd64" ;; +esac + +echo "==> Building trusted supervisor helper runtime" +STAGED_SUPERVISOR="${ROOT}/deploy/docker/.build/prebuilt-binaries/${DOCKER_ARCH}/openshell-sandbox" +RUNTIME_IMAGE="openshell-vm-helper-runtime:${DOCKER_ARCH}-$$" +mkdir -p "$(dirname "${STAGED_SUPERVISOR}")" +cp "${SUPERVISOR_BIN}" "${STAGED_SUPERVISOR}" + +case "$(uname -m)" in + aarch64|arm64) HOST_DOCKER_ARCH="arm64" ;; + x86_64|amd64) HOST_DOCKER_ARCH="amd64" ;; + *) HOST_DOCKER_ARCH="" ;; +esac + +if [ "${HOST_DOCKER_ARCH}" = "${DOCKER_ARCH}" ]; then + docker build \ + --build-arg "TARGETARCH=${DOCKER_ARCH}" \ + --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ + --tag "${RUNTIME_IMAGE}" \ + "${ROOT}" +else + docker buildx build \ + --load \ + --platform "linux/${DOCKER_ARCH}" \ + --build-arg "TARGETARCH=${DOCKER_ARCH}" \ + --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ + --tag "${RUNTIME_IMAGE}" \ + "${ROOT}" +fi + +RUNTIME_CONTAINER="$(docker create "${RUNTIME_IMAGE}")" +cleanup_runtime_image() { + docker rm -f "${RUNTIME_CONTAINER}" >/dev/null 2>&1 || true + docker image rm "${RUNTIME_IMAGE}" >/dev/null 2>&1 || true +} +trap cleanup_runtime_image EXIT +docker cp "${RUNTIME_CONTAINER}:/openshell-runtime" - \ + | zstd -19 -T0 -f -o "${SUPERVISOR_RUNTIME_OUTPUT}" +cleanup_runtime_image +trap - EXIT + echo "==> Bundled supervisor ready" echo " Binary: $(du -sh "${SUPERVISOR_BIN}" | cut -f1)" echo " Compressed: $(du -sh "${SUPERVISOR_OUTPUT}" | cut -f1)" +echo " Helper runtime: $(du -sh "${SUPERVISOR_RUNTIME_OUTPUT}" | cut -f1)" From dac36e629b8b044a5ccf8eab7a1124772d3afe4e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 2 Sep 2026 22:23:25 -0700 Subject: [PATCH 05/37] test(sandbox): match empty mode validation Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 2f29068712..5d426677ff 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -952,7 +952,7 @@ mod tests { #[test] fn mode_rejects_empty_value() { let err = "".parse::().unwrap_err(); - assert!(err.contains("at least one")); + assert!(err.contains("--mode must enable one of")); } #[cfg(target_os = "linux")] From f820eca6e959399c3caab0b665ba893ec1f269a4 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 2 Sep 2026 22:27:27 -0700 Subject: [PATCH 06/37] fix(isolation): gate Linux vsock imports Signed-off-by: Drew Newberry --- crates/openshell-isolation-interface/src/remote.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index 6c0528080f..781ba42ca7 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -5,7 +5,9 @@ #![allow(unsafe_code)] +#[cfg(target_os = "linux")] use std::mem::size_of; +#[cfg(target_os = "linux")] use std::os::fd::{FromRawFd as _, IntoRawFd as _}; use std::path::PathBuf; use std::sync::Arc; From 7b980516104187c9a5cbc26b48c716536e5b2950 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 2 Sep 2026 22:35:54 -0700 Subject: [PATCH 07/37] fix(network): gate Linux-only proxy imports Signed-off-by: Drew Newberry --- crates/openshell-supervisor-network/src/proxy.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 5848cdbfd9..b65ea8f908 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -44,7 +44,9 @@ use tokio::io::{ AsyncBufReadExt, AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, AsyncWriteExt, }; -use tokio::net::{TcpListener, TcpStream}; +use tokio::net::TcpListener; +#[cfg(any(target_os = "linux", test))] +use tokio::net::TcpStream; use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{debug, warn}; From bbf84d41e20d061b69d31e2edd6b1e9347c0e0df Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 2 Sep 2026 22:39:27 -0700 Subject: [PATCH 08/37] fix(process): gate Linux-only SSH imports Signed-off-by: Drew Newberry --- crates/openshell-supervisor-process/src/ssh.rs | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 2db6e7c66a..346eac7bac 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -16,7 +16,9 @@ use miette::{IntoDiagnostic, Result}; use nix::pty::{Winsize, openpty}; use nix::unistd::setsid; use openshell_core::VERSION; -use openshell_core::net::{connect_tcp_nodelay_best_effort, set_tcp_nodelay_best_effort}; +use openshell_core::net::connect_tcp_nodelay_best_effort; +#[cfg(target_os = "linux")] +use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::policy::SandboxPolicy; use openshell_ocsf::{ ActionId, ActivityId, DispositionId, SeverityId, SshActivityBuilder, StatusId, ocsf_emit, @@ -1563,13 +1565,6 @@ pub(crate) mod unsafe_pty { clippy::unnecessary_wraps, reason = "keeps pre-exec installation fallible for callers if setup gains validation" )] - #[cfg_attr( - not(target_os = "linux"), - allow( - clippy::unnecessary_wraps, - reason = "Linux pre_exec setup can fail while non-Linux setup cannot." - ) - )] pub fn install_pre_exec( cmd: &mut Command, policy: SandboxPolicy, @@ -1610,13 +1605,6 @@ pub(crate) mod unsafe_pty { clippy::unnecessary_wraps, reason = "keeps pre-exec installation fallible for callers if setup gains validation" )] - #[cfg_attr( - not(target_os = "linux"), - allow( - clippy::unnecessary_wraps, - reason = "Linux pre_exec setup can fail while non-Linux setup cannot." - ) - )] pub fn install_pre_exec_no_pty( cmd: &mut Command, policy: SandboxPolicy, From 7efdfd51b87a898090f0e510a5f32f7fbda62774 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 00:03:07 -0700 Subject: [PATCH 09/37] feat(isolation): add TLS boundary transport Signed-off-by: Drew Newberry --- Cargo.lock | 7 + .../openshell-isolation-interface/Cargo.toml | 4 + .../src/boundary_protocol.rs | 15 + .../src/remote.rs | 238 +++++++++++++++ crates/openshell-sandbox/Cargo.toml | 3 + .../openshell-sandbox/src/boundary_server.rs | 283 ++++++++++++++++++ 6 files changed, 550 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index bd89c34f9f..f94f979cab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4178,10 +4178,14 @@ dependencies = [ "async-trait", "libc", "openshell-core", + "rcgen", + "rustls 0.23.38", + "rustls-pemfile", "serde", "serde_json", "thiserror 2.0.18", "tokio", + "tokio-rustls 0.26.4", ] [[package]] @@ -4302,12 +4306,15 @@ dependencies = [ "openshell-supervisor-process", "prost", "prost-types", + "rcgen", "rustls", + "rustls-pemfile", "serde", "serde_json", "temp-env", "tempfile", "tokio", + "tokio-rustls 0.26.4", "tokio-tungstenite 0.26.2", "tonic", "tracing", diff --git a/crates/openshell-isolation-interface/Cargo.toml b/crates/openshell-isolation-interface/Cargo.toml index e86ba7e5e8..43e91a6986 100644 --- a/crates/openshell-isolation-interface/Cargo.toml +++ b/crates/openshell-isolation-interface/Cargo.toml @@ -17,11 +17,15 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +rustls = { workspace = true } +rustls-pemfile = { workspace = true } +tokio-rustls = { workspace = true } [target.'cfg(unix)'.dependencies] libc = "0.2" [dev-dependencies] +rcgen = { workspace = true } tokio = { workspace = true } [lints] diff --git a/crates/openshell-isolation-interface/src/boundary_protocol.rs b/crates/openshell-isolation-interface/src/boundary_protocol.rs index 0f26c1056f..5961ad004b 100644 --- a/crates/openshell-isolation-interface/src/boundary_protocol.rs +++ b/crates/openshell-isolation-interface/src/boundary_protocol.rs @@ -45,6 +45,14 @@ pub enum BoundaryTransport { Unix { socket_path: PathBuf }, /// Authenticated TCP endpoint, normally scoped by the runtime network. Tcp { address: std::net::SocketAddr }, + /// Confidential TCP endpoint authenticated with a driver-provisioned CA. + TlsTcp { + address: std::net::SocketAddr, + /// DNS identity expected in the boundary's server certificate. + server_name: String, + /// PEM-encoded CA certificate used only for this boundary. + ca_certificate_pem: String, + }, /// Linux host `AF_VSOCK` connection to a guest boundary. Vsock { guest_cid: u32, control_port: u32 }, } @@ -58,6 +66,13 @@ pub enum BoundaryListener { /// TCP listener address. An unspecified IP is valid for the boundary-side /// bind; the control-side transport must contain a concrete dial target. Tcp { address: std::net::SocketAddr }, + /// TLS-protected TCP listener. Drivers provision these files inside the + /// boundary; the private key is never included in the control topology. + TlsTcp { + address: std::net::SocketAddr, + certificate_chain_path: PathBuf, + private_key_path: PathBuf, + }, /// Guest `AF_VSOCK` listener. Vsock { control_port: u32 }, } diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index 781ba42ca7..62d45e2960 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -138,6 +138,19 @@ fn validate_topology( )); } } + BoundaryTransport::TlsTcp { + address, + server_name, + ca_certificate_pem, + } => { + validate_tcp_address(*address)?; + rustls::pki_types::ServerName::try_from(server_name.clone()).map_err(|error| { + BackendError::Descriptor(format!( + "boundary TLS server name {server_name:?} is invalid: {error}" + )) + })?; + tls_client_config(ca_certificate_pem)?; + } BoundaryTransport::Vsock { guest_cid, control_port, @@ -153,6 +166,38 @@ fn validate_topology( Ok(()) } +fn validate_tcp_address(address: std::net::SocketAddr) -> Result<(), BackendError> { + if address.port() == 0 || address.ip().is_unspecified() { + Err(BackendError::Descriptor( + "boundary TCP address must have a concrete IP and nonzero port".to_string(), + )) + } else { + Ok(()) + } +} + +fn tls_client_config(ca_certificate_pem: &str) -> Result { + let certificates = rustls_pemfile::certs(&mut ca_certificate_pem.as_bytes()) + .collect::, _>>() + .map_err(|error| { + BackendError::Descriptor(format!("parse boundary TLS CA certificate: {error}")) + })?; + if certificates.is_empty() { + return Err(BackendError::Descriptor( + "boundary TLS CA certificate PEM contains no certificates".to_string(), + )); + } + let mut roots = rustls::RootCertStore::empty(); + for certificate in certificates { + roots.add(certificate).map_err(|error| { + BackendError::Descriptor(format!("load boundary TLS CA certificate: {error}")) + })?; + } + Ok(rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth()) +} + fn validate_socket_path(path: &std::path::Path) -> Result<(), BackendError> { if path.is_absolute() { Ok(()) @@ -818,6 +863,37 @@ impl BoundaryClient { })?; Ok(Box::new(stream)) } + BoundaryTransport::TlsTcp { + address, + server_name, + ca_certificate_pem, + } => { + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[*address]) + .await + .map_err(|error| { + BackendError::Unavailable(format!( + "connect to boundary TLS endpoint {address}: {error}" + )) + })?; + let server_name = rustls::pki_types::ServerName::try_from(server_name.clone()) + .map_err(|error| { + BackendError::Descriptor(format!( + "boundary TLS server name {server_name:?} is invalid: {error}" + )) + })?; + let connector = tokio_rustls::TlsConnector::from(Arc::new(tls_client_config( + ca_certificate_pem, + )?)); + let stream = connector + .connect(server_name, stream) + .await + .map_err(|error| { + BackendError::Unavailable(format!( + "authenticate boundary TLS endpoint {address}: {error}" + )) + })?; + Ok(Box::new(stream)) + } BoundaryTransport::Vsock { guest_cid, control_port, @@ -925,9 +1001,96 @@ mod tests { use openshell_core::policy::{ FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, }; + use rcgen::generate_simple_self_signed; use super::*; + struct TestCertificate { + ca_pem: String, + server_config: Arc, + } + + fn test_certificate(name: &str) -> TestCertificate { + let certified = + generate_simple_self_signed(vec![name.to_string()]).expect("generate test certificate"); + let certificate = certified.cert.der().clone(); + let private_key = + rustls::pki_types::PrivatePkcs8KeyDer::from(certified.key_pair.serialize_der()); + let server_config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![certificate], private_key.into()) + .expect("build test TLS server config"); + TestCertificate { + ca_pem: certified.cert.pem(), + server_config: Arc::new(server_config), + } + } + + fn tls_topology( + address: std::net::SocketAddr, + server_name: &str, + ca_certificate_pem: String, + token: &str, + ) -> BoundaryTopology { + BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-1".to_string(), + transport: BoundaryTransport::TlsTcp { + address, + server_name: server_name.to_string(), + ca_certificate_pem, + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + bootstrap_token: token.to_string(), + } + } + + async fn spawn_tls_boundary( + certificate: Arc, + expected_token: String, + ) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test TLS boundary"); + let address = listener.local_addr().expect("read test listener address"); + let task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept TLS control client"); + let Ok(mut stream) = tokio_rustls::TlsAcceptor::from(certificate) + .accept(stream) + .await + else { + return; + }; + let declared_u32 = stream.read_u32().await.expect("read request length"); + let declared = declared_u32 as usize; + let mut frame = vec![0_u8; 4 + declared]; + frame[..4].copy_from_slice(&declared_u32.to_be_bytes()); + stream + .read_exact(&mut frame[4..]) + .await + .expect("read request frame"); + let request: RequestEnvelope = decode_frame(&frame).expect("decode request"); + let response = if request.boundary_id == "sandbox-1" + && request.bootstrap_token == expected_token + { + Response::Confirmed + } else { + Response::Error { + kind: "denied".to_string(), + message: "control authentication failed".to_string(), + } + }; + let frame = encode_frame(&ResponseEnvelope { + request_id: request.request_id, + response, + }) + .expect("encode response"); + stream.write_all(&frame).await.expect("write response"); + }); + (address, task) + } + fn sandbox() -> SandboxContext { SandboxContext { sandbox_id: "sandbox-1".to_string(), @@ -1015,4 +1178,79 @@ mod tests { }; validate_topology(&topology, &sandbox()).expect("TCP topology should be valid"); } + + #[test] + fn topology_rejects_invalid_tls_configuration() { + let topology = tls_topology( + "127.0.0.1:5500".parse().expect("valid address"), + "not a dns name!", + "not a certificate".to_string(), + "0123456789abcdef0123456789abcdef", + ); + assert!(matches!( + validate_topology(&topology, &sandbox()), + Err(BackendError::Descriptor(_)) + )); + } + + #[tokio::test] + async fn tls_tcp_round_trip_verifies_server_certificate() { + let certificate = test_certificate("boundary.test"); + let (address, server) = spawn_tls_boundary(certificate.server_config, "a".repeat(32)).await; + let client = BoundaryClient::new(tls_topology( + address, + "boundary.test", + certificate.ca_pem, + &"a".repeat(32), + )); + + assert_eq!( + client.call(Request::Confirm).await.expect("TLS request"), + Response::Confirmed + ); + server.await.expect("TLS test server"); + } + + #[tokio::test] + async fn tls_tcp_preserves_boundary_token_authentication() { + let certificate = test_certificate("boundary.test"); + let (address, server) = spawn_tls_boundary( + certificate.server_config, + "expected-token-expected-token-12".to_string(), + ) + .await; + let client = BoundaryClient::new(tls_topology( + address, + "boundary.test", + certificate.ca_pem, + "incorrect-token-incorrect-token", + )); + + assert!(matches!( + client.call(Request::Confirm).await, + Err(BackendError::Denied(_)) + )); + server.await.expect("TLS test server"); + } + + #[tokio::test] + async fn tls_tcp_rejects_an_untrusted_server_certificate() { + let presented = test_certificate("boundary.test"); + let trusted = test_certificate("boundary.test"); + let (address, server) = spawn_tls_boundary(presented.server_config, "a".repeat(32)).await; + let client = BoundaryClient::new(tls_topology( + address, + "boundary.test", + trusted.ca_pem, + &"a".repeat(32), + )); + + assert!(matches!( + client.connect_boundary_once().await, + Err(BackendError::Unavailable(_)) + )); + // The server observes the client's fatal alert and may fail its accept; + // completing the task is sufficient for this rejection test. + let _ = server.await; + } } diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index d9636b2eba..4b7cd5e99c 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -43,6 +43,8 @@ nix = { workspace = true } # TLS crypto provider install (main.rs) rustls = { workspace = true } +rustls-pemfile = { workspace = true } +tokio-rustls = { workspace = true } # Serialization (serde_json::json! for OCSF unmapped fields) base64 = { workspace = true } @@ -81,6 +83,7 @@ telemetry = ["openshell-core/telemetry"] bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] [dev-dependencies] +rcgen = { workspace = true } tempfile = "3" temp-env = "0.3" tokio-tungstenite = { workspace = true } diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index eea84891c3..fff5423758 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -135,11 +135,25 @@ mod linux { BoundaryListenerConfig::Tcp { address } if address.port() == 0 => { return Err("boundary TCP listener port must be nonzero".to_string()); } + BoundaryListenerConfig::TlsTcp { + address, + certificate_chain_path, + private_key_path, + } if address.port() == 0 + || !certificate_chain_path.is_absolute() + || !private_key_path.is_absolute() => + { + return Err( + "boundary TLS listener requires a nonzero port and absolute certificate paths" + .to_string(), + ); + } BoundaryListenerConfig::Vsock { control_port: 0 } => { return Err("boundary control port must be nonzero".to_string()); } BoundaryListenerConfig::Unix { .. } | BoundaryListenerConfig::Tcp { .. } + | BoundaryListenerConfig::TlsTcp { .. } | BoundaryListenerConfig::Vsock { .. } => {} } match &config.agent_identity { @@ -204,6 +218,13 @@ mod linux { Ok(stream) => { let runtime = runtime.clone(); std::thread::spawn(move || { + let stream = match stream.establish(&runtime.process_runtime) { + Ok(stream) => stream, + Err(error) => { + tracing::warn!(%error, "Boundary control transport handshake failed"); + return; + } + }; if let Err(error) = serve_one(stream, &runtime) { tracing::warn!(%error, "Boundary control request failed"); } @@ -1167,6 +1188,10 @@ mod linux { Vsock(OwnedFd), Unix(std::os::unix::net::UnixListener), Tcp(std::net::TcpListener), + TlsTcp { + listener: std::net::TcpListener, + server_config: Arc, + }, } impl ControlListener { @@ -1190,6 +1215,20 @@ mod linux { listener.set_nonblocking(true)?; Ok(Self::Tcp(listener)) } + BoundaryListenerConfig::TlsTcp { + address, + certificate_chain_path, + private_key_path, + } => { + let listener = std::net::TcpListener::bind(address)?; + listener.set_nonblocking(true)?; + let server_config = + load_tls_server_config(certificate_chain_path, private_key_path)?; + Ok(Self::TlsTcp { + listener, + server_config: Arc::new(server_config), + }) + } } } @@ -1235,6 +1274,17 @@ mod linux { Ok(fd) } + #[cfg(test)] + fn tcp_local_addr(&self) -> io::Result { + match self { + Self::Tcp(listener) | Self::TlsTcp { listener, .. } => listener.local_addr(), + Self::Unix(_) | Self::Vsock(_) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "control listener is not TCP", + )), + } + } + fn accept(&self) -> io::Result { match self { Self::Vsock(fd) => { @@ -1263,17 +1313,91 @@ mod linux { } Ok(ControlStream::Tcp(stream)) } + Self::TlsTcp { + listener, + server_config, + } => { + let (stream, _) = listener.accept()?; + Ok(ControlStream::PendingTlsTcp { + stream, + server_config: server_config.clone(), + }) + } } } } + fn load_tls_server_config( + certificate_chain_path: &Path, + private_key_path: &Path, + ) -> io::Result { + let certificate_bytes = std::fs::read(certificate_chain_path)?; + let certificates = rustls_pemfile::certs(&mut certificate_bytes.as_slice()) + .collect::, _>>()?; + if certificates.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "boundary TLS certificate chain contains no certificates", + )); + } + let private_key_bytes = std::fs::read(private_key_path)?; + let private_key = rustls_pemfile::private_key(&mut private_key_bytes.as_slice())? + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "boundary TLS private-key file contains no private key", + ) + })?; + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, private_key) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error)) + } + enum ControlStream { Vsock(File), Unix(std::os::unix::net::UnixStream), Tcp(std::net::TcpStream), + PendingTlsTcp { + stream: std::net::TcpStream, + server_config: Arc, + }, + TlsTcp { + stream: Option>>, + runtime: tokio::runtime::Handle, + }, } impl ControlStream { + fn establish(self, runtime: &tokio::runtime::Handle) -> io::Result { + let Self::PendingTlsTcp { + stream, + server_config, + } = self + else { + return Ok(self); + }; + stream.set_nonblocking(true)?; + let stream = { + let _guard = runtime.enter(); + tokio::net::TcpStream::from_std(stream)? + }; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + let acceptor = tokio_rustls::TlsAcceptor::from(server_config); + let stream = runtime.block_on(async { + tokio::time::timeout(CONTROL_IO_TIMEOUT, acceptor.accept(stream)) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS handshake timed out") + })? + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) + })?; + Ok(Self::TlsTcp { + stream: Some(Box::new(stream)), + runtime: runtime.clone(), + }) + } + fn set_timeout(&self, timeout: Duration) -> io::Result<()> { if let Self::Unix(stream) = self { stream.set_read_timeout(Some(timeout))?; @@ -1283,6 +1407,15 @@ mod linux { stream.set_read_timeout(Some(timeout))?; return stream.set_write_timeout(Some(timeout)); } + if matches!(self, Self::TlsTcp { .. }) { + return Ok(()); + } + if matches!(self, Self::PendingTlsTcp { .. }) { + return Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )); + } let Self::Vsock(file) = self else { unreachable!("Unix and TCP streams returned above") }; @@ -1340,6 +1473,12 @@ mod linux { openshell_core::net::set_tcp_nodelay_best_effort(&stream); Ok(Box::new(stream)) } + Self::TlsTcp { mut stream, .. } => Ok(stream + .take() + .expect("boundary TLS stream can only be converted once")), + Self::PendingTlsTcp { .. } => { + Err("boundary TLS stream has not completed its handshake".to_string()) + } Self::Vsock(file) => { let stream = unsafe { std::os::unix::net::UnixStream::from_raw_fd(file.into_raw_fd()) }; @@ -1361,6 +1500,23 @@ mod linux { Self::Vsock(stream) => stream.read(buffer), Self::Unix(stream) => stream.read(buffer), Self::Tcp(stream) => stream.read(buffer), + Self::TlsTcp { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .read(buffer), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS read timed out") + })? + }), + Self::PendingTlsTcp { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), } } } @@ -1371,6 +1527,23 @@ mod linux { Self::Vsock(stream) => stream.write(buffer), Self::Unix(stream) => stream.write(buffer), Self::Tcp(stream) => stream.write(buffer), + Self::TlsTcp { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .write(buffer), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS write timed out") + })? + }), + Self::PendingTlsTcp { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), } } @@ -1379,6 +1552,23 @@ mod linux { Self::Vsock(stream) => stream.flush(), Self::Unix(stream) => stream.flush(), Self::Tcp(stream) => stream.flush(), + Self::TlsTcp { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .flush(), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS flush timed out") + })? + }), + Self::PendingTlsTcp { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), } } } @@ -1386,6 +1576,7 @@ mod linux { #[cfg(test)] mod tests { use super::*; + use rcgen::generate_simple_self_signed; #[test] fn boundary_config_debug_redacts_token() { @@ -1453,6 +1644,98 @@ mod linux { validate_config(&config).unwrap(); } + + #[test] + fn tls_listener_preserves_session_when_control_switches_to_async_streaming() { + let directory = tempfile::tempdir().expect("temporary directory"); + let certificate_path = directory.path().join("boundary.crt"); + let private_key_path = directory.path().join("boundary.key"); + let certified = generate_simple_self_signed(vec!["boundary.test".to_string()]) + .expect("generate test certificate"); + std::fs::write(&certificate_path, certified.cert.pem()) + .expect("write test certificate"); + std::fs::write(&private_key_path, certified.key_pair.serialize_pem()) + .expect("write test private key"); + let listener = ControlListener::bind(&BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:0".parse().expect("valid address"), + certificate_chain_path: certificate_path, + private_key_path, + }) + .expect("bind TLS listener"); + let address = listener.tcp_local_addr().expect("TLS listener address"); + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test runtime"); + let server_runtime = runtime.handle().clone(); + let server = std::thread::spawn(move || { + let mut stream = loop { + match listener.accept() { + Ok(stream) => break stream, + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + std::thread::yield_now(); + } + Err(error) => panic!("accept TLS stream: {error}"), + } + } + .establish(&server_runtime) + .expect("establish TLS stream"); + let mut first = [0_u8; 4]; + Read::read_exact(&mut stream, &mut first).expect("read blocking TLS phase"); + assert_eq!(&first, b"sync"); + Write::write_all(&mut stream, b"ack1").expect("write blocking TLS phase"); + server_runtime.block_on(async move { + let mut stream = stream.into_tokio().expect("convert negotiated TLS stream"); + let mut second = [0_u8; 5]; + stream + .read_exact(&mut second) + .await + .expect("read async TLS phase"); + assert_eq!(&second, b"async"); + stream + .write_all(b"ack2") + .await + .expect("write async TLS phase"); + }); + }); + + runtime.block_on(async { + let mut roots = rustls::RootCertStore::empty(); + roots + .add(certified.cert.der().clone()) + .expect("trust test certificate"); + let client_config = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + let stream = tokio::net::TcpStream::connect(address) + .await + .expect("connect TLS listener"); + let server_name = rustls::pki_types::ServerName::try_from("boundary.test") + .expect("valid server name"); + let mut stream = tokio_rustls::TlsConnector::from(Arc::new(client_config)) + .connect(server_name, stream) + .await + .expect("verify TLS listener"); + stream.write_all(b"sync").await.expect("write first phase"); + let mut first_ack = [0_u8; 4]; + stream + .read_exact(&mut first_ack) + .await + .expect("read first acknowledgement"); + assert_eq!(&first_ack, b"ack1"); + stream + .write_all(b"async") + .await + .expect("write second phase"); + let mut second_ack = [0_u8; 4]; + stream + .read_exact(&mut second_ack) + .await + .expect("read second acknowledgement"); + assert_eq!(&second_ack, b"ack2"); + }); + server.join().expect("TLS boundary server thread"); + } } } From efefbe526689861db40775acc0d6062b3296b74a Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 00:27:45 -0700 Subject: [PATCH 10/37] fix(isolation): retain access after main exit Signed-off-by: Drew Newberry --- .../openshell-sandbox/src/boundary_server.rs | 75 +++++++++++------ crates/openshell-sandbox/src/lib.rs | 81 ++++++++++++++++++- .../src/boundary_io.rs | 21 +++++ .../src/delegated.rs | 6 +- 4 files changed, 154 insertions(+), 29 deletions(-) diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index fff5423758..9c491e6f47 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -53,8 +53,14 @@ mod linux { }; const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30); + static BOUNDARY_TERMINATION_REQUESTED: AtomicBool = AtomicBool::new(false); + + extern "C" fn request_boundary_termination(_signal: libc::c_int) { + BOUNDARY_TERMINATION_REQUESTED.store(true, Ordering::Release); + } pub fn run_boundary(config_path: &Path) -> Result<(), String> { + install_boundary_signal_handlers()?; if std::process::id() == 1 { prepare_pid1_filesystems()?; } @@ -90,6 +96,26 @@ mod linux { serve(&config.listener, runtime) } + fn install_boundary_signal_handlers() -> Result<(), String> { + BOUNDARY_TERMINATION_REQUESTED.store(false, Ordering::Release); + let action = nix::sys::signal::SigAction::new( + nix::sys::signal::SigHandler::Handler(request_boundary_termination), + nix::sys::signal::SaFlags::empty(), + nix::sys::signal::SigSet::empty(), + ); + for signal in [ + nix::sys::signal::Signal::SIGTERM, + nix::sys::signal::Signal::SIGINT, + ] { + // SAFETY: the installed handler only performs a lock-free atomic + // store, which is async-signal-safe, and remains valid for the + // lifetime of the boundary process. + unsafe { nix::sys::signal::sigaction(signal, &action) } + .map_err(|error| format!("install boundary {signal:?} handler: {error}"))?; + } + Ok(()) + } + fn protect_boundary_config_file(config_path: &Path) -> Result<(), String> { let path = CString::new(config_path.as_os_str().as_bytes()).map_err(|error| { format!( @@ -211,7 +237,8 @@ mod linux { .map_err(|error| format!("bind boundary control listener: {error}"))?; tracing::info!(?config, "Boundary control listener ready"); loop { - if runtime.is_shutdown_requested() { + if BOUNDARY_TERMINATION_REQUESTED.load(Ordering::Acquire) { + runtime.shutdown(); return Ok(()); } match listener.accept() { @@ -349,19 +376,12 @@ mod linux { } _ => {} } - let shutdown_after_response = matches!( - &request.request, - Request::Wait { process_id } if runtime.is_main_process(process_id) - ); let response = ResponseEnvelope { request_id: request.request_id, response: runtime.dispatch(request), }; write_frame(&mut stream, &response) .map_err(|error| format!("write control frame: {error}"))?; - if shutdown_after_response && matches!(response.response, Response::Exited { .. }) { - runtime.request_shutdown(); - } Ok(()) } @@ -375,7 +395,6 @@ mod linux { attached_policy: Mutex>, next_exec_id: AtomicU64, exec_handles: Mutex>, - shutdown_requested: AtomicBool, } struct ExecHandle { @@ -498,24 +517,22 @@ mod linux { attached_policy: Mutex::new(None), next_exec_id: AtomicU64::new(1), exec_handles: Mutex::new(std::collections::HashMap::new()), - shutdown_requested: AtomicBool::new(false), } } - fn is_shutdown_requested(&self) -> bool { - self.shutdown_requested.load(Ordering::Acquire) - } - - fn request_shutdown(&self) { - self.shutdown_requested.store(true, Ordering::Release); - } - - fn is_main_process(&self, process_id: &str) -> bool { - let state = lock(&self.state); - matches!( - &*state, - RuntimeState::Running(process) if process.process_id() == process_id - ) + fn shutdown(&self) { + let process = { + let state = lock(&self.state); + match &*state { + RuntimeState::Running(process) => Some(process.clone()), + RuntimeState::AwaitingAttach + | RuntimeState::Bound(_) + | RuntimeState::Ready(_) => None, + } + }; + if let Some(process) = process { + process.boundary_runtime.deactivate(); + } } fn dispatch(&self, envelope: RequestEnvelope) -> Response { @@ -955,6 +972,7 @@ mod linux { main_session: Arc, attached: AtomicBool, _netns: Option>, + boundary_runtime: Arc, } struct ManagedProcessLaunch { @@ -1008,7 +1026,7 @@ mod linux { prepared.netns.as_deref(), None, None, - Some(boundary_runtime), + Some(boundary_runtime.clone()), None, )) .map_err(|error| format!("start process supervisor leaf: {error:?}"))?; @@ -1042,6 +1060,7 @@ mod linux { main_session, attached: AtomicBool::new(false), _netns: prepared.netns, + boundary_runtime, }) } @@ -1099,6 +1118,12 @@ mod linux { } } + impl Drop for ManagedProcess { + fn drop(&mut self) { + self.boundary_runtime.deactivate(); + } + } + async fn bridge_main_stream( stream: openshell_isolation_interface::contract::BoundaryDuplexStream, session: Arc, diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index c0c1e89d59..1b6c7ffb2e 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -79,6 +79,37 @@ pub fn run_boundary(config_path: &std::path::Path) -> Result<()> { boundary_server::run_boundary(config_path).map_err(|error| miette::miette!(error)) } +async fn retain_remote_access_plane( + proxy_exited: impl Future, + shutdown_requested: impl Future, +) -> Result<()> { + tokio::pin!(proxy_exited); + tokio::pin!(shutdown_requested); + tokio::select! { + () = &mut proxy_exited => Err(miette::miette!( + "control-mode proxy accept loop exited unexpectedly" + )), + () = &mut shutdown_requested => Ok(()), + } +} + +#[cfg(unix)] +async fn wait_for_control_shutdown_signal() { + use tokio::signal::unix::{SignalKind, signal}; + + let mut sigterm = signal(SignalKind::terminate()).expect("install control SIGTERM handler"); + let mut sigint = signal(SignalKind::interrupt()).expect("install control SIGINT handler"); + tokio::select! { + _ = sigterm.recv() => {} + _ = sigint.recv() => {} + } +} + +#[cfg(not(unix))] +async fn wait_for_control_shutdown_signal() { + let _ = tokio::signal::ctrl_c().await; +} + use openshell_core::denial::DenialEvent; use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; use openshell_core::proposals::AgentProposals; @@ -979,6 +1010,8 @@ pub async fn run_sandbox( info!(backend = %backend_name, "Control-mode access plane started"); let instance_id = boundary_access.instance_id().to_string(); let wait_agent = agent.clone(); + let shutdown_requested = wait_for_control_shutdown_signal(); + tokio::pin!(shutdown_requested); let wait = async move { wait_agent .wait() @@ -993,14 +1026,33 @@ pub async fn run_sandbox( }) .map_err(|error| miette::miette!(error.to_string())) }; - let exit_code = tokio::select! { - result = wait => result?, + let (exit_code, retain_access) = tokio::select! { + result = wait => (result?, true), () = &mut proxy_exited => { let _ = agent.terminate().await; return Err(miette::miette!( "control-mode proxy accept loop exited unexpectedly" )); } + () = &mut shutdown_requested => { + let _ = agent + .signal(openshell_isolation_interface::contract::BoundarySignal::Term) + .await; + let status = if let Ok(result) = timeout(Duration::from_secs(5), agent.wait()).await { + result + } else { + let _ = agent.terminate().await; + agent.wait().await + } + .map_err(|error| miette::miette!(error.to_string()))?; + let exit_code = match status { + openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code) => code, + openshell_isolation_interface::contract::BoundaryExitStatus::Signaled(signal) => { + 128_i32.saturating_add(signal) + } + }; + (exit_code, false) + } }; if let (Some(endpoint), Some(id)) = (openshell_endpoint.as_deref(), sandbox_id.as_deref()) { openshell_supervisor_process::delegated::report_main_process_completion( @@ -1012,6 +1064,10 @@ pub async fn run_sandbox( .await; } boundary_access.finish_main(exit_code).await; + if retain_access { + info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); + retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; + } drop(running); drop(boundary_access); exit_code @@ -4645,6 +4701,27 @@ fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String mod tests { use super::*; + #[tokio::test] + async fn remote_access_plane_outlives_main_completion_until_teardown() { + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let retained = retain_remote_access_plane(std::future::pending(), async { + let _ = shutdown_rx.await; + }); + tokio::pin!(retained); + + assert!( + timeout(Duration::from_millis(10), &mut retained) + .await + .is_err(), + "access plane must remain live after canonical process completion" + ); + shutdown_tx.send(()).expect("request teardown"); + timeout(Duration::from_secs(1), &mut retained) + .await + .expect("teardown should release retained access plane") + .expect("clean teardown"); + } + #[test] fn transparent_tcp_capability_requires_exact_driver_marker() { let required = openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY; diff --git a/crates/openshell-supervisor-process/src/boundary_io.rs b/crates/openshell-supervisor-process/src/boundary_io.rs index c477cc7b90..adc8550d0b 100644 --- a/crates/openshell-supervisor-process/src/boundary_io.rs +++ b/crates/openshell-supervisor-process/src/boundary_io.rs @@ -310,4 +310,25 @@ mod tests { runtime.unregister_process_group(pid, &second_terminal); assert_eq!(runtime.registered_process_group_count(), 0); } + + #[test] + fn canonical_process_completion_does_not_end_boundary_runtime() { + let runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); + let terminal = Arc::new(std::sync::atomic::AtomicBool::new(true)); + runtime + .register_process_group(42, terminal.clone(), Arc::new(Mutex::new(()))) + .expect("register canonical process"); + + runtime.unregister_process_group(42, &terminal); + + runtime + .ensure_active() + .expect("canonical completion must preserve exec and forwarding"); + assert_eq!(runtime.registered_process_group_count(), 0); + runtime.deactivate(); + assert!(matches!( + runtime.ensure_active(), + Err(BackendError::Terminated(_)) + )); + } } diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 8dcceead64..5f4fe147a5 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -404,7 +404,10 @@ impl SpawnedAgent { self.main_session.clone() } - /// Wait for exit, enforcing the admitted wall-clock timeout. + /// Wait for the canonical process to exit, enforcing its admitted + /// wall-clock timeout. Completion does not end the boundary: exec and + /// loopback forwarding remain available until the boundary owner tears + /// down the retained runtime. pub async fn wait(&mut self) -> Result { let signaler = self.signaler(); let status = if self.timeout_secs == 0 { @@ -421,7 +424,6 @@ impl SpawnedAgent { }; self.boundary_runtime .unregister_process_group(self.handle.pid(), &self.terminal); - self.boundary_runtime.deactivate(); let _ = self.main_session.finish(status.code(), false).await; self.main_session.mark_terminal_reported(); Ok(status) From aaf064b3219dc8a2b10b4caf7b5e78d436c31283 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 00:39:51 -0700 Subject: [PATCH 11/37] test(isolation): cover access after canonical exit Signed-off-by: Drew Newberry --- .../openshell-sandbox/src/boundary_server.rs | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 9c491e6f47..39e2805fd2 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -1761,6 +1761,134 @@ mod linux { }); server.join().expect("TLS boundary server thread"); } + + #[test] + fn canonical_exit_preserves_pending_network_accept_and_exec() { + const CHILD_MARKER: &str = "OPENSHELL_TEST_RETAINED_BOUNDARY_CHILD"; + if std::env::var_os(CHILD_MARKER).is_none() { + let status = std::process::Command::new( + std::env::current_exe().expect("current test executable"), + ) + .args([ + "--exact", + "boundary_server::linux::tests::canonical_exit_preserves_pending_network_accept_and_exec", + "--nocapture", + ]) + .env(CHILD_MARKER, "1") + .status() + .expect("run isolated retained-boundary test"); + assert!(status.success(), "isolated retained-boundary test failed"); + return; + } + + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test process runtime"); + let network_listener = process_runtime + .block_on(tokio::net::TcpListener::bind("127.0.0.1:0")) + .expect("bind mediated network listener"); + let network_address = network_listener + .local_addr() + .expect("mediated network listener address"); + let policy = openshell_core::policy::SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy { + mode: openshell_core::policy::NetworkMode::Proxy, + proxy: Some(openshell_core::policy::ProxyPolicy { + http_addr: Some("127.0.0.1:3128".parse().expect("proxy address")), + }), + }, + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }; + let prepared = PreparedBoundary { + netns: None, + network_listener: Some(Arc::new(network_listener)), + proxy_port: 3128, + }; + let process = Arc::new( + ManagedProcess::spawn( + process_runtime.handle(), + ManagedProcessLaunch { + sandbox_id: "sandbox-retained".to_string(), + spec: AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: None, + timeout_secs: 5, + interactive: false, + }, + policy, + resolved_identity: ResolvedProcessIdentity::new( + Some(nix::unistd::Uid::current().as_raw()), + Some(nix::unistd::Gid::current().as_raw()), + ), + provider_env: std::collections::HashMap::new(), + ca_file_paths: None, + }, + prepared, + ) + .expect("spawn canonical process"), + ); + let boundary = Arc::new(BoundaryRuntime::new( + BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-retained".to_string(), + bootstrap_token: "a".repeat(32), + protect_config_file: false, + listener: BoundaryListenerConfig::Tcp { + address: "127.0.0.1:5500".parse().expect("control address"), + }, + resource_claims: std::collections::BTreeMap::new(), + agent_identity: BoundaryAgentIdentity::None, + trusted_runtime_root: "/tmp".into(), + child_env: std::collections::HashMap::new(), + }, + process_runtime.handle().clone(), + )); + *lock(&boundary.state) = RuntimeState::Running(process.clone()); + + let accept_boundary = boundary; + let pending_accept = std::thread::spawn(move || accept_boundary.accept_network()); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !process.has_exited() && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(process.has_exited(), "canonical process did not exit"); + + let _workload_connection = + std::net::TcpStream::connect(network_address).expect("connect mediated listener"); + let (_accepted, _identity) = pending_accept + .join() + .expect("pending AcceptNetwork thread") + .expect("AcceptNetwork remains viable after canonical exit"); + + let mut session = process_runtime + .block_on( + process.boundary_exec().exec( + ExecSpecWire { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), "printf retained".to_string()], + env: Vec::new(), + workdir: None, + pty: false, + } + .into(), + ), + ) + .expect("exec after canonical exit"); + let mut output = String::new(); + process_runtime + .block_on(session.stdout.read_to_string(&mut output)) + .expect("read retained exec output"); + assert_eq!(output, "retained"); + assert!(matches!( + process_runtime.block_on(session.process.wait()), + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(0)) + )); + } } } From fcd06547a61c9791bc410a44ddc370defbdc454e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 01:04:31 -0700 Subject: [PATCH 12/37] fix(isolation): contextualize delegated startup failures Signed-off-by: Drew Newberry --- .../openshell-supervisor-process/src/delegated.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 5f4fe147a5..3aeddc0aeb 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -12,7 +12,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::time::Duration; -use miette::{IntoDiagnostic as _, Result}; +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use openshell_core::policy::{NetworkMode, SandboxPolicy}; use openshell_core::proposals::AgentProposals; use openshell_core::provider_credentials::ProviderCredentialState; @@ -225,8 +225,10 @@ pub async fn spawn_workload( crate::run::install_initial_agent_skill(sandbox_id, openshell_endpoint, &agent_proposals).await; #[cfg(target_os = "linux")] - crate::process::prepare_supervisor_identity_mount_namespace_from_env()?; - crate::sandbox::apply_supervisor_startup_hardening()?; + crate::process::prepare_supervisor_identity_mount_namespace_from_env() + .wrap_err("prepare supervisor identity mount namespace")?; + crate::sandbox::apply_supervisor_startup_hardening() + .wrap_err("apply supervisor startup hardening")?; #[cfg(target_os = "linux")] let bypass_handle = netns.and_then(|namespace| { @@ -245,7 +247,7 @@ pub async fn spawn_workload( } else { crate::process::RuntimePidLimitMode::Warn }; - crate::process::check_runtime_pid_limit(mode)?; + crate::process::check_runtime_pid_limit(mode).wrap_err("check runtime PID limit")?; } #[cfg(target_os = "linux")] @@ -301,7 +303,8 @@ pub async fn spawn_workload( netns, ca_file_paths.as_ref(), &provider_env, - )?; + ) + .wrap_err("spawn delegated workload process")?; #[cfg(not(target_os = "linux"))] let mut handle = ProcessHandle::spawn( program, From 0176eb8cacc3ecc89339909050ba0ad36bdeb9ab Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 01:10:06 -0700 Subject: [PATCH 13/37] fix(isolation): contextualize boundary bootstrap failures Signed-off-by: Drew Newberry --- crates/openshell-supervisor-process/src/delegated.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 3aeddc0aeb..7cd4fbf238 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -211,15 +211,19 @@ pub async fn spawn_workload( #[cfg(unix)] if enforcement_mode.uses_privileged_process_setup() { - crate::process::update_sandbox_passwd_entries()?; - crate::process::validate_sandbox_user_with_identity(policy, resolved_process_identity)?; - crate::process::validate_sandbox_group_with_identity(policy, resolved_process_identity)?; + crate::process::update_sandbox_passwd_entries() + .wrap_err("update sandbox passwd entries")?; + crate::process::validate_sandbox_user_with_identity(policy, resolved_process_identity) + .wrap_err("validate sandbox process user")?; + crate::process::validate_sandbox_group_with_identity(policy, resolved_process_identity) + .wrap_err("validate sandbox process group")?; crate::process::prepare_filesystem_with_identity( policy, resolved_process_identity, workspace.root(), workspace.home().is_some(), - )?; + ) + .wrap_err("prepare delegated workload filesystem")?; } crate::run::install_initial_agent_skill(sandbox_id, openshell_endpoint, &agent_proposals).await; From 63ce03f8618faaa5d7a36839a8908aa6905dcb1b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 01:15:49 -0700 Subject: [PATCH 14/37] docs(rfc): align boundary transport and lifetime Signed-off-by: Drew Newberry --- architecture/compute-runtimes.md | 8 +++---- architecture/sandbox.md | 14 ++++++++--- rfc/0012-isolation-backend/README.md | 6 ++--- rfc/0012-isolation-backend/topology-matrix.md | 23 +++++++++++-------- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index a69406a39f..f81563a4bf 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -17,7 +17,7 @@ is responsible for: - Supplying TLS or secret material for supervisor callbacks. - Providing the supervisor binary or image in the workload. - Provisioning protected control and boundary configs plus a private Unix socket, - authenticated TCP, or vsock transport when the supervisor is separated. + TLS-authenticated TCP, or vsock transport when the supervisor is separated. Runtime-specific code supplies immutable resource claims and transport coordinates; the shared boundary protocol supplies lifecycle, exec, signaling, forwarding, and binary identity semantics. @@ -253,10 +253,10 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| -| Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | +| Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Creates the workload with Docker `network_mode=none`; a host control process mediates egress and access over a private bind-mounted Unix socket. | | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | -| Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | -| VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | +| Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | The proxy-pod topology fences workload egress with NetworkPolicy, runs boundary mode as the workload entrypoint, and runs control mode in a separate zero-capability pod over per-boundary TLS. It requires an enforcing CNI and trusted sandbox namespace. | +| VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | The guest is NIC-less and runs boundary mode as PID 1; host control owns gateway networking and reaches the guest over vsock. The driver persists each accepted launch request and writable overlay for restart. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through diff --git a/architecture/sandbox.md b/architecture/sandbox.md index fc5957deb3..39b3c9e1e0 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -28,7 +28,9 @@ Separated runtimes split the same logical supervisor into two placements: | `openshell-sandbox --mode=boundary --boundary-config ` | Runs inside or adjacent to the workload. Owns process launch and observation, exec signaling, PTY control, loopback forwarding, egress capture, and binary identity. It has no gateway credentials or independent policy authority. | The compute driver provisions the two protected configurations and their private -Unix-socket, authenticated TCP, or vsock transport. All configurations carry the +Unix-socket, TLS-authenticated TCP, or vsock transport. TCP across a shared or +operator-managed network uses a driver-provisioned trust root and verified server +name; NetworkPolicy alone is not a confidentiality boundary. All configurations carry the same boundary ID, bootstrap credential, protocol version, and immutable driver resource claims. The shared remote backend authenticates each request and rejects a resource-claim mismatch. Driver crates do not appear in generic supervisor lifecycle, network, @@ -47,13 +49,19 @@ workload when its host ownership could otherwise coincide with the workload UID. gateway, depending on mode. 3. The isolation backend attaches and confirms standing enforcement. Boundary mode prepares boundary-local filesystem, process, and egress controls. -4. Control mode starts the admitted main process through the backend. Boundary +4. Control mode connects network mediation, then confirms enforcement and starts + the admitted main process through the backend. Boundary mode applies launch-time controls before its first untrusted instruction. -5. Control mode starts the policy proxy and local SSH server. Exec and loopback +5. Control mode starts the local SSH server. Exec and loopback streams cross the shared boundary protocol when the modes are separated. 6. Control mode opens a supervisor session back to the gateway for connect, exec, file sync, config polling, and log push. +When the admitted main process exits, its status and retained terminal output +remain available. The confirmed boundary and control-owned access plane continue +to serve policy-authorized exec and loopback forwarding until explicit stop or +delete tears down the boundary and terminates any remaining workload processes. + ## Isolation Layers OpenShell uses overlapping controls rather than a single sandbox primitive: diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md index c46b435896..4a1a6c3360 100644 --- a/rfc/0012-isolation-backend/README.md +++ b/rfc/0012-isolation-backend/README.md @@ -91,7 +91,7 @@ Co-located deployments may keep the existing in-process backend and omit boundar `openshell-sandbox --mode=boundary --boundary-config ` runs inside, or immediately adjacent to, the execution environment. The driver supplies the config over a protected filesystem channel. It names one boundary, one private listener, one per-boundary bootstrap credential, and the workload identity. The identity is either a platform-resolved numeric UID/GID pair or an OCI `Config.User` declaration that boundary mode resolves against the workload filesystem. Boundary mode authenticates and scopes every request to that boundary. It never accepts policy or identity claims from workload code and it cannot authorize an operation independently of control mode. -The shared protocol is versioned independently of a driver's resource descriptor. It carries `attach`, `confirm`, `start_agent`, agent and exec `wait`/`signal`/`terminate`, PTY resize, loopback-only forwarding, and attributed egress streams. Drivers may use a private Unix socket, authenticated TCP scoped by the runtime network, a Unix endpoint mapped to guest vsock, or host `AF_VSOCK`; adding a transport does not change lifecycle or operation semantics. Secrets are delivered in protected files, redacted from debug output, and never placed in workload-visible environment or process arguments. When a bind-mounted file's host owner may match the workload UID, the driver requires boundary mode to re-own it as root-only before the first workload instruction. Unix socket inodes permit cross-UID control within their private driver-owned directory; the per-boundary bootstrap credential authenticates every request. +The shared protocol is versioned independently of a driver's resource descriptor. It carries `attach`, `confirm`, `start_agent`, agent and exec `wait`/`signal`/`terminate`, PTY resize, loopback-only forwarding, and attributed egress streams. Drivers may use a private Unix socket, TLS-authenticated TCP, a Unix endpoint mapped to guest vsock, or host `AF_VSOCK`; adding a transport does not change lifecycle or operation semantics. A TCP transport that crosses a shared or operator-managed network must authenticate the server name against a driver-provisioned trust root and encrypt every control and stream request. Network isolation alone is not a confidentiality boundary. Secrets are delivered in protected files, redacted from debug output, and never placed in workload-visible environment or process arguments. When a bind-mounted file's host owner may match the workload UID, the driver requires boundary mode to re-own it as root-only before the first workload instruction. Unix socket inodes permit cross-UID control within their private driver-owned directory; the per-boundary bootstrap credential authenticates every request. A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver or external orchestrator owns the durable resource lifecycle even when the backend establishes a resource as part of `attach`. @@ -229,7 +229,7 @@ The states have normative meanings: `start_agent` is the sole operation that may make the admitted agent runnable. The backend may create or release the process, but workload-controlled code cannot run before `start_agent` applies the required controls. -`RunningBoundary::agent()` returns a handle for the admitted agent process. Processes started through `BoundaryExec` run in the same boundary and have their own process handles. Every workload process remains within the provisioned execution environment. Any exit of the admitted agent ends `Running`; the backend then terminates every remaining workload process within that environment and rejects further runtime operations, except `wait` as defined below. +`RunningBoundary::agent()` returns a handle for the admitted agent process. Processes started through `BoundaryExec` run in the same boundary and have their own process handles. Every workload process remains within the provisioned execution environment. Exit of the admitted agent produces a stable `wait` result and ends that process generation, but it does not implicitly tear down the boundary. The control role may continue serving terminal output, `exec`, and loopback forwarding within the same confirmed boundary until the compute driver or control role explicitly tears the boundary down. Explicit teardown terminates every remaining workload process and rejects new runtime operations. ### Runtime operations @@ -369,7 +369,7 @@ Failures resolve as follows: - network-mediation errors yield no authorized connection and do not by themselves end `Running`; and - retained runtime handles and the network-mediation source reject new operations whenever the boundary ends, except `BoundaryProcess::wait` where the backend can still return its stable result. -Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the compute driver reclaims or deprovisions the topology. If normal backend cleanup is unavailable, the compute driver uses the topology's trusted cleanup path to terminate the execution environment and invalidate the binding before reclaim or reuse. On normal agent exit, `BoundaryProcess::wait` returns the stable exit status. A retained `wait` result may outlive teardown. +Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the compute driver reclaims or deprovisions the topology. If normal backend cleanup is unavailable, the compute driver uses the topology's trusted cleanup path to terminate the execution environment and invalidate the binding before reclaim or reuse. Normal agent exit alone does not end the boundary: `BoundaryProcess::wait` returns the stable exit status while the confirmed access plane remains available until explicit teardown. A retained `wait` result may outlive teardown. ### Topologies diff --git a/rfc/0012-isolation-backend/topology-matrix.md b/rfc/0012-isolation-backend/topology-matrix.md index bd5a8167e0..85335be545 100644 --- a/rfc/0012-isolation-backend/topology-matrix.md +++ b/rfc/0012-isolation-backend/topology-matrix.md @@ -9,15 +9,20 @@ kernel; it does not select a deployment or establish conformance. | Pattern | Control and network-mediation placement | Boundary-mode placement | Workload-kernel relationship | Topology status | |---|---|---|---|---| | **Co-located/in-pod** | With the workload; the legacy in-process backend may omit boundary mode | Same process when used | Trusted components share the workload's host kernel | Placement implemented (original topology) | -| **Kubernetes proxy pod** | Trusted proxy pod | Trusted sidecar or companion sharing workload PID and network observation | Shared node kernel; pod security boundaries separate control from workload | Prototype in #3144 | -| **Docker** | Gateway host | Trusted container entrypoint sharing the workload container's PID and network namespaces | Shared host kernel | Prototype in #2965 | -| **MicroVM** | Gateway host | Guest PID 1 | Boundary mode shares the guest kernel; control is kernel-separated | Prototype in #2945 | - -The Kubernetes proxy-pod topology uses the same authenticated boundary protocol -as Docker and VM. Kubernetes-specific code provisions pod placement, the -private transport, and immutable pod/Sandbox resource claims. It still requires -mediated DNS and attributed connections; pod readiness or the existence of a -`NetworkPolicy` object alone does not confirm enforcement. +| **Kubernetes proxy pod** | Trusted control pod | Boundary-mode workload entrypoint owning the workload PID and network namespaces | Shared cluster-node kernel; pod security boundaries separate control from workload | Implemented in #3144; requires a conforming NetworkPolicy CNI and trusted namespace | +| **Docker** | Gateway host | Trusted container entrypoint sharing the workload container's PID and network namespaces | Shared host kernel | Implemented in #2965 | +| **MicroVM** | Gateway host | Guest PID 1 | Boundary mode shares the guest kernel; control is kernel-separated | Implemented in #2945 | + +The Kubernetes proxy-pod topology uses the same boundary protocol as Docker and +VM, with per-boundary TLS because the connection traverses the pod network. +Kubernetes-specific code provisions the workload fence, pair labels, boundary +Service, control Deployment, immutable bootstrap Secret, and stable +namespace/Sandbox/Deployment/NetworkPolicy claims. The workload pod has no +direct egress; attributed proxy streams cross the TLS channel and hostname +resolution occurs on the control side. Admission requires an explicitly acknowledged conforming CNI and a +namespace in which untrusted principals cannot create pods, mutate pair labels, +or read bootstrap Secrets. Pod readiness or the existence of a `NetworkPolicy` +object alone does not prove enforcement. ## Durable rules From 14f74608ff9b6e21edd9dcaad8ff2d614e5a9be8 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 01:28:14 -0700 Subject: [PATCH 15/37] fix(isolation): reconnect running boundary controls Signed-off-by: Drew Newberry --- architecture/sandbox.md | 6 +- .../openshell-sandbox/src/boundary_server.rs | 292 ++++++++++++++++-- rfc/0012-isolation-backend/README.md | 2 + 3 files changed, 273 insertions(+), 27 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 39b3c9e1e0..1d70757cf8 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -554,7 +554,11 @@ engine with a gateway policy revision. to new connections or the next parsed HTTP request where the proxy can safely re-evaluate. - If the supervisor relay drops, the sandbox can keep running, but connect and - exec operations fail until the supervisor registers again. + exec operations fail until the supervisor registers again. In a separated + topology, a replacement control process replays the identical boundary + lifecycle and receives the existing process handle. The boundary rejects + changed launch inputs and releases the single main-process attachment when + the old control transport closes. - If the canonical main process exits, the supervisor durably reports the normalized result immediately. A foreground create declares a one-shot main attachment, so the supervisor accepts it even after a fast process exits, diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 39e2805fd2..10dc89c6e5 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -309,7 +309,7 @@ mod linux { return runtime.stream_exec(stream, &process_id, session); } Request::AttachProcess { process_id } => { - let (session, terminal) = match runtime.attach_process(&process_id) { + let (attachment, terminal) = match runtime.attach_process(&process_id) { Ok(attachment) => attachment, Err(response) => { return write_frame( @@ -330,7 +330,7 @@ mod linux { }, ) .map_err(|error| format!("write process attachment response: {error}"))?; - return runtime.stream_process(stream, session); + return runtime.stream_process(stream, attachment); } Request::PortForward { host, port } => { let target = LoopbackTarget::new(host, port) @@ -393,6 +393,10 @@ mod linux { /// carrying a different policy is denied instead of silently keeping /// the first policy. attached_policy: Mutex>, + /// The complete launch request accepted by the boundary. A replacement + /// control process may replay it after reconnecting, but may not change + /// any launch input or start a second workload. + started_agent: Mutex>, next_exec_id: AtomicU64, exec_handles: Mutex>, } @@ -402,6 +406,27 @@ mod linux { terminal: Option>, } + #[derive(Clone, PartialEq, Eq)] + struct StartedAgent { + sandbox_id: String, + spec: AgentSpecWire, + policy: SandboxPolicyWire, + ca_cert: Option>, + ca_bundle: Option>, + provider_env: std::collections::HashMap, + } + + struct MainAttachment { + process: Arc, + session: Arc, + } + + impl Drop for MainAttachment { + fn drop(&mut self) { + self.process.attached.store(false, Ordering::Release); + } + } + enum RuntimeState { AwaitingAttach, Bound(PreparedBoundary), @@ -515,6 +540,7 @@ mod linux { process_runtime, state: Mutex::new(RuntimeState::AwaitingAttach), attached_policy: Mutex::new(None), + started_agent: Mutex::new(None), next_exec_id: AtomicU64::new(1), exec_handles: Mutex::new(std::collections::HashMap::new()), } @@ -715,26 +741,31 @@ mod linux { bridged } - fn attach_process(&self, process_id: &str) -> Result<(Arc, bool), Response> { + fn attach_process(&self, process_id: &str) -> Result<(MainAttachment, bool), Response> { let process = self.running_process(process_id)?; - if process.attached.swap(true, Ordering::AcqRel) { + if process + .attached + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { return Err(guest_error( "denied", "main process already has a control attachment", )); } let session = process.main_session(); - Ok((session.clone(), session.terminal())) + let terminal = session.terminal(); + Ok((MainAttachment { process, session }, terminal)) } fn stream_process( &self, stream: ControlStream, - session: Arc, + attachment: MainAttachment, ) -> Result<(), String> { self.process_runtime.block_on(async move { let stream = stream.into_tokio()?; - bridge_main_stream(stream, session).await + bridge_main_stream(stream, attachment.session.clone()).await }) } @@ -753,7 +784,7 @@ mod linux { *state = RuntimeState::Bound(prepared); Response::Attached } - RuntimeState::Bound(_) => { + RuntimeState::Bound(_) | RuntimeState::Ready(_) | RuntimeState::Running(_) => { // Idempotent retry of the same attach; a different policy // must not be silently coalesced onto the bound boundary. if lock(&self.attached_policy).as_ref() == Some(&policy) { @@ -762,7 +793,6 @@ mod linux { guest_error("denied", "attach policy does not match the bound boundary") } } - _ => guest_error("invalid", "boundary has already advanced past attach"), } } @@ -776,13 +806,10 @@ mod linux { *state = RuntimeState::Ready(prepared.clone()); Response::Confirmed } - RuntimeState::Ready(_) => Response::Confirmed, + RuntimeState::Ready(_) | RuntimeState::Running(_) => Response::Confirmed, RuntimeState::AwaitingAttach => { guest_error("invalid", "boundary must be attached before confirm") } - RuntimeState::Running(_) => { - guest_error("invalid", "boundary has already started its agent") - } } } @@ -796,6 +823,26 @@ mod linux { provider_env: std::collections::HashMap, ) -> Response { let mut state = lock(&self.state); + let requested = StartedAgent { + sandbox_id: sandbox_id.clone(), + spec: spec.clone(), + policy: policy.clone(), + ca_cert: ca_cert.clone(), + ca_bundle: ca_bundle.clone(), + provider_env: provider_env.clone(), + }; + if let RuntimeState::Running(process) = &*state { + return if lock(&self.started_agent).as_ref() == Some(&requested) { + Response::Started { + process_id: process.process_id(), + } + } else { + guest_error( + "denied", + "start_agent inputs do not match the running boundary", + ) + }; + } let RuntimeState::Ready(prepared) = &*state else { return guest_error("invalid", "boundary must be confirmed before start_agent"); }; @@ -832,6 +879,7 @@ mod linux { Err(error) => return guest_error("failed", error), }; let process_id = process.process_id(); + *lock(&self.started_agent) = Some(requested); *state = RuntimeState::Running(process); Response::Started { process_id } } @@ -1132,13 +1180,24 @@ mod linux { let writer = Arc::new(tokio::sync::Mutex::new(writer)); let (owner, input) = session.acquire_input().map_err(str::to_string)?; let mut output = session.subscribe(); - let input_task = tokio::spawn(async move { + let mut input_task = tokio::spawn(async move { + let mut input = Some(input); while let Some((channel, payload)) = read_stream_frame(&mut reader).await? { match channel { - STREAM_STDIN => input.send(payload).await.map_err(|_| { - io::Error::new(io::ErrorKind::BrokenPipe, "main process stdin closed") - })?, - STREAM_STDIN_CLOSED => break, + STREAM_STDIN => { + let Some(input) = input.as_ref() else { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "main process stdin already closed", + )); + }; + input.send(payload).await.map_err(|_| { + io::Error::new(io::ErrorKind::BrokenPipe, "main process stdin closed") + })?; + } + // Keep reading after stdin closes so transport EOF still + // releases this control process's attachment lease. + STREAM_STDIN_CLOSED => drop(input.take()), _ => { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -1150,7 +1209,17 @@ mod linux { Ok::<(), io::Error>(()) }); let result = loop { - let (channel, payload) = match output.recv().await { + let output_message = tokio::select! { + input_result = &mut input_task => { + break match input_result { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(format!("read main process attachment: {error}")), + Err(error) => Err(format!("join main process input stream: {error}")), + }; + } + output_message = output.recv() => output_message, + }; + let (channel, payload) = match output_message { Ok(MainOutput::Stdout(payload)) => (STREAM_STDOUT, payload.to_vec()), Ok(MainOutput::Stderr(payload)) => (STREAM_STDERR, payload.to_vec()), Ok(MainOutput::Exit(code)) => { @@ -1762,6 +1831,148 @@ mod linux { server.join().expect("TLS boundary server thread"); } + #[test] + fn control_restart_replays_running_lifecycle_exactly_once() { + const CHILD_MARKER: &str = "OPENSHELL_TEST_BOUNDARY_RECONNECT_CHILD"; + if std::env::var_os(CHILD_MARKER).is_none() { + let status = std::process::Command::new( + std::env::current_exe().expect("current test executable"), + ) + .args([ + "--exact", + "boundary_server::linux::tests::control_restart_replays_running_lifecycle_exactly_once", + "--nocapture", + ]) + .env(CHILD_MARKER, "1") + .status() + .expect("run isolated reconnect test"); + assert!(status.success(), "isolated reconnect test failed"); + return; + } + + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test process runtime"); + let boundary = Arc::new(BoundaryRuntime::new( + BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: "sandbox-reconnect".to_string(), + bootstrap_token: "a".repeat(32), + protect_config_file: false, + listener: BoundaryListenerConfig::Tcp { + address: "127.0.0.1:5500".parse().expect("control address"), + }, + resource_claims: std::collections::BTreeMap::new(), + agent_identity: BoundaryAgentIdentity::Resolved { + uid: nix::unistd::Uid::current().as_raw(), + gid: nix::unistd::Gid::current().as_raw(), + }, + trusted_runtime_root: "/tmp".into(), + child_env: std::collections::HashMap::new(), + }, + process_runtime.handle().clone(), + )); + let policy = SandboxPolicyWire::from(openshell_core::policy::SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }); + let spec = AgentSpecWire { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + workdir: None, + timeout_secs: 60, + interactive: false, + }; + + assert_eq!(boundary.attach(policy.clone()), Response::Attached); + assert_eq!(boundary.confirm(), Response::Confirmed); + let start = || { + boundary.start_agent( + "sandbox-reconnect".to_string(), + spec.clone(), + policy.clone(), + None, + None, + std::collections::HashMap::new(), + ) + }; + let Response::Started { process_id } = start() else { + panic!("initial start did not succeed"); + }; + + let (first_attachment, _) = boundary + .attach_process(&process_id) + .expect("initial main-process attachment"); + assert!(boundary.attach_process(&process_id).is_err()); + let (boundary_stream, control_stream) = + std::os::unix::net::UnixStream::pair().expect("main attachment socket pair"); + let stream_boundary = boundary.clone(); + let stream_thread = std::thread::spawn(move || { + stream_boundary + .stream_process(ControlStream::Unix(boundary_stream), first_attachment) + }); + drop(control_stream); + stream_thread + .join() + .expect("join disconnected main attachment") + .expect("transport EOF cleanly ends main attachment"); + let (replacement_attachment, _) = boundary + .attach_process(&process_id) + .expect("replacement main-process attachment after disconnect"); + drop(replacement_attachment); + + assert_eq!(boundary.attach(policy.clone()), Response::Attached); + assert_eq!(boundary.confirm(), Response::Confirmed); + assert_eq!( + start(), + Response::Started { + process_id: process_id.clone() + } + ); + + let mut changed_policy = policy.clone(); + changed_policy.version += 1; + assert!(matches!( + boundary.attach(changed_policy.clone()), + Response::Error { kind, .. } if kind == "denied" + )); + assert!(matches!( + boundary.start_agent( + "sandbox-reconnect".to_string(), + spec, + changed_policy, + None, + None, + std::collections::HashMap::new(), + ), + Response::Error { kind, .. } if kind == "denied" + )); + + let (_exec_id, mut exec) = boundary + .start_exec(ExecSpecWire { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), "printf reconnected".to_string()], + env: Vec::new(), + workdir: None, + pty: false, + }) + .expect("exec after reconnect"); + let mut output = String::new(); + process_runtime + .block_on(exec.stdout.read_to_string(&mut output)) + .expect("read reconnect exec output"); + assert_eq!(output, "reconnected"); + assert!(matches!( + process_runtime.block_on(exec.process.wait()), + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(0)) + )); + assert_eq!(boundary.terminate(&process_id), Response::Terminated); + } + #[test] fn canonical_exit_preserves_pending_network_accept_and_exec() { const CHILD_MARKER: &str = "OPENSHELL_TEST_RETAINED_BOUNDARY_CHILD"; @@ -1808,18 +2019,20 @@ mod linux { network_listener: Some(Arc::new(network_listener)), proxy_port: 3128, }; + let agent_spec = AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: None, + timeout_secs: 5, + interactive: false, + }; + let wire_policy = SandboxPolicyWire::from(policy.clone()); let process = Arc::new( ManagedProcess::spawn( process_runtime.handle(), ManagedProcessLaunch { sandbox_id: "sandbox-retained".to_string(), - spec: AgentSpecWire { - program: "/bin/true".to_string(), - args: Vec::new(), - workdir: None, - timeout_secs: 5, - interactive: false, - }, + spec: agent_spec.clone(), policy, resolved_identity: ResolvedProcessIdentity::new( Some(nix::unistd::Uid::current().as_raw()), @@ -1849,6 +2062,33 @@ mod linux { process_runtime.handle().clone(), )); *lock(&boundary.state) = RuntimeState::Running(process.clone()); + *lock(&boundary.attached_policy) = Some(wire_policy.clone()); + *lock(&boundary.started_agent) = Some(StartedAgent { + sandbox_id: "sandbox-retained".to_string(), + spec: agent_spec.clone(), + policy: wire_policy.clone(), + ca_cert: None, + ca_bundle: None, + provider_env: std::collections::HashMap::new(), + }); + + // A replacement control process replays the durable lifecycle and + // receives the original process rather than spawning another one. + assert_eq!(boundary.attach(wire_policy.clone()), Response::Attached); + assert_eq!(boundary.confirm(), Response::Confirmed); + assert_eq!( + boundary.start_agent( + "sandbox-retained".to_string(), + agent_spec, + wire_policy, + None, + None, + std::collections::HashMap::new(), + ), + Response::Started { + process_id: process.process_id() + } + ); let accept_boundary = boundary; let pending_accept = std::thread::spawn(move || accept_boundary.accept_network()); diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md index 4a1a6c3360..754f545397 100644 --- a/rfc/0012-isolation-backend/README.md +++ b/rfc/0012-isolation-backend/README.md @@ -229,6 +229,8 @@ The states have normative meanings: `start_agent` is the sole operation that may make the admitted agent runnable. The backend may create or release the process, but workload-controlled code cannot run before `start_agent` applies the required controls. +A separated boundary retains the complete accepted `attach` and `start_agent` inputs for the lifetime of `Running`. If its control process restarts, the replacement replays `attach`, `confirm`, and `start_agent` with the same authenticated boundary identity, resource claims, policy, and launch inputs. The boundary returns the existing process handle without starting a second workload. Any changed input is denied. A main-process stream has one active control owner; transport closure releases that attachment so the replacement control can attach. Compute drivers fence control replacement so old and new control processes do not overlap (for example, a Kubernetes control Deployment uses `Recreate`). + `RunningBoundary::agent()` returns a handle for the admitted agent process. Processes started through `BoundaryExec` run in the same boundary and have their own process handles. Every workload process remains within the provisioned execution environment. Exit of the admitted agent produces a stable `wait` result and ends that process generation, but it does not implicitly tear down the boundary. The control role may continue serving terminal output, `exec`, and loopback forwarding within the same confirmed boundary until the compute driver or control role explicitly tears the boundary down. Explicit teardown terminates every remaining workload process and rejects new runtime operations. ### Runtime operations From 9b60f3d42dcb80400b2690d12e3e9958166f2663 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 01:38:20 -0700 Subject: [PATCH 16/37] feat(supervisor): persist canonical completion marker Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/lib.rs | 59 ++++++++++++++++++++++++++++ crates/openshell-sandbox/src/main.rs | 33 ++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 1b6c7ffb2e..e851dfb1f7 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -27,6 +27,7 @@ mod sidecar_control; use miette::{IntoDiagnostic, Result, WrapErr}; use std::future::Future; +use std::io::Write as _; use std::pin::Pin; use std::sync::Arc; #[cfg(target_os = "linux")] @@ -171,6 +172,7 @@ pub async fn run_sandbox( upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, topology_descriptor: Option, admitted_isolation_backend: Option, + main_exit_marker: Option, ) -> Result { let (program, args) = command .split_first() @@ -1054,6 +1056,11 @@ pub async fn run_sandbox( (exit_code, false) } }; + if retain_access && let Some(marker) = main_exit_marker.as_deref() { + persist_main_exit_marker(marker, exit_code) + .into_diagnostic() + .wrap_err("persist canonical-process completion marker")?; + } if let (Some(endpoint), Some(id)) = (openshell_endpoint.as_deref(), sandbox_id.as_deref()) { openshell_supervisor_process::delegated::report_main_process_completion( endpoint, @@ -1365,6 +1372,41 @@ pub async fn run_sandbox( Ok(exit_code) } +fn persist_main_exit_marker(path: &std::path::Path, exit_code: i32) -> std::io::Result<()> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("completion marker has no parent: {}", path.display()), + ) + })?; + let name = path.file_name().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("completion marker has no file name: {}", path.display()), + ) + })?; + let temporary = parent.join(format!( + ".{}.tmp-{}", + name.to_string_lossy(), + std::process::id() + )); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + let mut file = options.open(&temporary)?; + writeln!(file, "exit_code={exit_code}")?; + file.sync_all()?; + std::fs::rename(&temporary, path)?; + std::fs::File::open(parent)?.sync_all() +} + /// Wait for SIGINT or SIGTERM. Used in network-only mode where there is /// no entrypoint child whose lifetime drives the supervisor's exit. async fn wait_for_shutdown_signal() { @@ -4701,6 +4743,23 @@ fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String mod tests { use super::*; + #[test] + fn main_exit_marker_atomically_replaces_previous_value() { + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("main-exited"); + std::fs::write(&marker, b"stale\n").unwrap(); + + persist_main_exit_marker(&marker, 23).unwrap(); + + assert_eq!(std::fs::read_to_string(&marker).unwrap(), "exit_code=23\n"); + assert!( + !directory + .path() + .join(format!(".main-exited.tmp-{}", std::process::id())) + .exists() + ); + } + #[tokio::test] async fn remote_access_plane_outlives_main_completion_until_teardown() { let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 5d426677ff..d625190029 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -274,6 +274,28 @@ struct Args { /// Driver-owned protected file holding the raw topology payload. #[arg(long)] topology_payload_file: Option, + + /// Driver-owned durable marker written after canonical process completion. + /// Used by separated control processes whose boundary remains alive. + #[arg(long, hide = true)] + main_exit_marker: Option, +} + +fn validate_main_exit_marker(mode: Mode, marker: Option<&Path>) -> Result<()> { + let Some(marker) = marker else { + return Ok(()); + }; + if !mode.control { + return Err(miette::miette!( + "--main-exit-marker is only valid with --mode=control" + )); + } + if !marker.is_absolute() { + return Err(miette::miette!( + "--main-exit-marker must be an absolute path" + )); + } + Ok(()) } /// Internal one-shot command used by the privileged supervisor to validate an @@ -580,6 +602,7 @@ fn main() -> Result<()> { } let args = Args::parse(); + validate_main_exit_marker(args.mode, args.main_exit_marker.as_deref())?; if args.mode.boundary { let console_filter = @@ -821,6 +844,7 @@ fn main() -> Result<()> { upstream_proxy_args, topology_descriptor, admitted_isolation_backend, + args.main_exit_marker, ) .await })?; @@ -955,6 +979,15 @@ mod tests { assert!(err.contains("--mode must enable one of")); } + #[test] + fn main_exit_marker_requires_control_mode_and_absolute_path() { + let control = "control".parse::().unwrap(); + let combined = DEFAULT_MODE.parse::().unwrap(); + assert!(validate_main_exit_marker(control, Some(Path::new("/state/exited"))).is_ok()); + assert!(validate_main_exit_marker(control, Some(Path::new("relative"))).is_err()); + assert!(validate_main_exit_marker(combined, Some(Path::new("/state/exited"))).is_err()); + } + #[cfg(target_os = "linux")] #[test] fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { From f23307713dbb3771ae8b21dc543a7cdf9ab3c346 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 01:45:53 -0700 Subject: [PATCH 17/37] fix(isolation): harden remote boundary lifecycle Signed-off-by: Drew Newberry --- .../src/provider_credentials.rs | 76 ++++++- .../src/boundary_protocol.rs | 22 +- .../src/remote.rs | 46 +++- .../openshell-sandbox/src/boundary_server.rs | 115 +++++++++- crates/openshell-sandbox/src/lib.rs | 15 +- .../src/delegated.rs | 23 +- .../openshell-supervisor-process/src/ssh.rs | 210 ++++++++++++++++-- 7 files changed, 461 insertions(+), 46 deletions(-) diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index 2b1537a21b..784f263ed2 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -359,6 +359,17 @@ impl ProviderCredentialState { /// here so SDKs can read them at startup. /// 3. Everything else stays as placeholders for proxy-time resolution. pub fn child_env_with_gcp_resolved(&self) -> HashMap { + self.child_env_snapshot_with_gcp_resolved().1 + } + + /// Return the current revision and its workload-facing environment from + /// one state snapshot. + /// + /// Remote isolation boundaries use the pair as a revisioned update. The + /// revision must describe the exact environment sent across the boundary, + /// so callers must not obtain the two values through separate lock + /// acquisitions. + pub fn child_env_snapshot_with_gcp_resolved(&self) -> (u64, HashMap) { use crate::google_cloud; let inner = self @@ -376,7 +387,7 @@ impl ProviderCredentialState { .any(|key| env.contains_key(*key) && inner.non_secret_environment_keys.contains(*key)); if !has_gcp_metadata && !has_gcp_config { - return env; + return (inner.current.revision, env); } if has_gcp_metadata { @@ -414,7 +425,41 @@ impl ProviderCredentialState { } } - env + (inner.current.revision, env) + } + + /// Install a newer workload-facing environment snapshot and return the + /// effective revision. Stale and duplicate updates are idempotently + /// ignored under the same write lock that installs the snapshot. + pub fn install_child_env_snapshot_if_newer( + &self, + revision: u64, + mut child_env: HashMap, + ) -> u64 { + let mut inner = self + .inner + .write() + .expect("provider credential state poisoned"); + if revision <= inner.current.revision { + return inner.current.revision; + } + + for key in &inner.suppressed_keys { + child_env.remove(key); + } + inner.current = Arc::new(ProviderCredentialSnapshot { + revision, + child_env, + dynamic_credentials: HashMap::new(), + }); + inner.generations.clear(); + inner.current_resolver = None; + inner.combined_resolver = None; + inner.non_secret_environment_keys.clear(); + inner.static_credential_bindings.clear(); + inner.known_static_credential_keys.clear(); + inner.static_credential_identity_epochs.clear(); + revision } /// Return the GCP token placeholder and its remaining lifetime in seconds. @@ -2117,6 +2162,33 @@ mod tests { ); } + #[test] + fn child_env_snapshot_update_is_monotonic_and_atomic() { + let state = ProviderCredentialState::from_child_env_snapshot( + 4, + HashMap::from([("TOKEN".to_string(), "four".to_string())]), + ); + + assert_eq!( + state.install_child_env_snapshot_if_newer( + 6, + HashMap::from([("TOKEN".to_string(), "six".to_string())]), + ), + 6 + ); + assert_eq!( + state.install_child_env_snapshot_if_newer( + 5, + HashMap::from([("TOKEN".to_string(), "stale".to_string())]), + ), + 6 + ); + + let (revision, env) = state.child_env_snapshot_with_gcp_resolved(); + assert_eq!(revision, 6); + assert_eq!(env.get("TOKEN").map(String::as_str), Some("six")); + } + #[test] fn stale_generation_falls_back_to_current_credential_after_retention_window() { let state = ProviderCredentialState::from_environment( diff --git a/crates/openshell-isolation-interface/src/boundary_protocol.rs b/crates/openshell-isolation-interface/src/boundary_protocol.rs index 5961ad004b..c439f402c4 100644 --- a/crates/openshell-isolation-interface/src/boundary_protocol.rs +++ b/crates/openshell-isolation-interface/src/boundary_protocol.rs @@ -35,7 +35,7 @@ pub const STREAM_EXIT: u8 = 3; pub const STREAM_STDIN_CLOSED: u8 = 4; pub const MAX_STREAM_FRAME_BYTES: usize = 64 * 1024; /// Version of the driver-to-supervisor boundary descriptor and config shape. -pub const BOUNDARY_PROTOCOL_VERSION: u32 = 1; +pub const BOUNDARY_PROTOCOL_VERSION: u32 = 2; /// Control-side endpoint for a driver-provisioned boundary. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -299,6 +299,11 @@ pub enum Request { policy: Box, ca_cert: Option>, ca_bundle: Option>, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + }, + UpdateProviderEnvironment { + revision: u64, provider_env: std::collections::HashMap, }, AttachProcess { @@ -351,6 +356,7 @@ impl fmt::Debug for Request { policy: _, ca_cert, ca_bundle, + provider_env_revision, provider_env, } => formatter .debug_struct("StartAgent") @@ -359,6 +365,18 @@ impl fmt::Debug for Request { .field("policy", &"") .field("ca_cert_present", &ca_cert.is_some()) .field("ca_bundle_present", &ca_bundle.is_some()) + .field("provider_env_revision", provider_env_revision) + .field( + "provider_env_keys", + &provider_env.keys().collect::>(), + ) + .finish(), + Self::UpdateProviderEnvironment { + revision, + provider_env, + } => formatter + .debug_struct("UpdateProviderEnvironment") + .field("revision", revision) .field( "provider_env_keys", &provider_env.keys().collect::>(), @@ -419,6 +437,7 @@ pub enum Response { Attached, Confirmed, Started { process_id: String }, + ProviderEnvironmentUpdated { revision: u64 }, ProcessAttached { terminal: bool }, Exited { status: ExitStatusWire }, Signaled, @@ -818,6 +837,7 @@ mod tests { })), ca_cert: Some(b"test certificate".to_vec()), ca_bundle: Some(b"test bundle".to_vec()), + provider_env_revision: 7, provider_env: std::collections::HashMap::from([( "OPENAI_API_KEY".to_string(), "test credential".to_string(), diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index 62d45e2960..caffcb1df1 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -47,19 +47,19 @@ const MIN_BOOTSTRAP_TOKEN_BYTES: usize = 32; pub struct RemoteIsolationBackend { backend_name: String, ca_file_paths: Arc>>, - provider_env: std::collections::HashMap, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, } impl RemoteIsolationBackend { pub fn new( backend_name: impl Into, ca_file_paths: Arc>>, - provider_env: std::collections::HashMap, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, ) -> Self { Self { backend_name: backend_name.into(), ca_file_paths, - provider_env, + provider_credentials, } } } @@ -102,7 +102,7 @@ impl IsolationBackend for RemoteIsolationBackend { mediation: Arc::new(RemoteNetworkMediation { client }), host_gateway_ip, ca_file_paths: self.ca_file_paths.clone(), - provider_env: self.provider_env.clone(), + provider_credentials: self.provider_credentials.clone(), })) } } @@ -226,7 +226,7 @@ struct RemoteBound { mediation: Arc, host_gateway_ip: Option, ca_file_paths: Arc>>, - provider_env: std::collections::HashMap, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, } #[async_trait] @@ -250,7 +250,7 @@ impl BoundBoundary for RemoteBound { policy: self.policy, sandbox_id: self.sandbox_id, ca_file_paths: self.ca_file_paths, - provider_env: self.provider_env, + provider_credentials: self.provider_credentials, })) } } @@ -261,7 +261,7 @@ struct RemoteReady { policy: openshell_core::policy::SandboxPolicy, sandbox_id: String, ca_file_paths: Arc>>, - provider_env: std::collections::HashMap, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, } #[async_trait] @@ -286,6 +286,9 @@ impl ReadyBoundary for RemoteReady { } else { (None, None) }; + let (provider_env_revision, provider_env) = self + .provider_credentials + .child_env_snapshot_with_gcp_resolved(); let response = self .client .call(Request::StartAgent { @@ -294,7 +297,8 @@ impl ReadyBoundary for RemoteReady { policy: Box::new(SandboxPolicyWire::from(self.policy)), ca_cert, ca_bundle, - provider_env: self.provider_env, + provider_env_revision, + provider_env, }) .await?; let Response::Started { process_id } = response else { @@ -308,6 +312,7 @@ impl ReadyBoundary for RemoteReady { process, exec: Arc::new(RemoteExec { client: self.client.clone(), + provider_credentials: self.provider_credentials, }), port_forward: Arc::new(RemotePortForward { client: self.client, @@ -461,11 +466,36 @@ async fn pump_process_responses( struct RemoteExec { client: Arc, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, } #[async_trait] impl BoundaryExec for RemoteExec { async fn exec(&self, spec: ExecSpec) -> Result { + let (revision, provider_env) = self + .provider_credentials + .child_env_snapshot_with_gcp_resolved(); + let response = self + .client + .call_idempotent(Request::UpdateProviderEnvironment { + revision, + provider_env, + }) + .await?; + let Response::ProviderEnvironmentUpdated { + revision: effective_revision, + } = response + else { + return Err(unexpected_response( + "provider_environment_updated", + &response, + )); + }; + if effective_revision < revision { + return Err(BackendError::Process(format!( + "boundary provider environment revision {effective_revision} is older than requested revision {revision}" + ))); + } open_exec_session(self.client.clone(), spec).await } } diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 10dc89c6e5..dca358ee98 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -295,7 +295,7 @@ mod linux { .map_err(|error| format!("write exec error response: {error}")); } }; - write_frame( + if let Err(error) = write_frame( &mut stream, &ResponseEnvelope { request_id: request.request_id, @@ -304,8 +304,10 @@ mod linux { pty: session.terminal.is_some(), }, }, - ) - .map_err(|error| format!("write exec start response: {error}"))?; + ) { + runtime.cancel_exec_start(&process_id, session.process.clone()); + return Err(format!("write exec start response: {error}")); + } return runtime.stream_exec(stream, &process_id, session); } Request::AttachProcess { process_id } => { @@ -413,6 +415,7 @@ mod linux { policy: SandboxPolicyWire, ca_cert: Option>, ca_bundle: Option>, + provider_env_revision: u64, provider_env: std::collections::HashMap, } @@ -586,8 +589,21 @@ mod linux { policy, ca_cert, ca_bundle, + provider_env_revision, provider_env, - } => self.start_agent(sandbox_id, spec, *policy, ca_cert, ca_bundle, provider_env), + } => self.start_agent( + sandbox_id, + spec, + *policy, + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + ), + Request::UpdateProviderEnvironment { + revision, + provider_env, + } => self.update_provider_environment(revision, provider_env), Request::Wait { process_id } => self.wait(&process_id), Request::Signal { process_id, signal } => self.signal(&process_id, signal), Request::Terminate { process_id } => self.terminate(&process_id), @@ -652,6 +668,14 @@ mod linux { } } + fn cancel_exec_start(&self, process_id: &str, process: Arc) { + lock(&self.exec_handles).remove(process_id); + self.process_runtime.block_on(async move { + let _ = process.terminate().await; + let _ = process.wait().await; + }); + } + fn resize_process(&self, process_id: &str, cols: u16, rows: u16) -> Response { if let Ok(process) = self.running_process(process_id) { let session = process.main_session(); @@ -813,6 +837,7 @@ mod linux { } } + #[allow(clippy::too_many_arguments)] fn start_agent( &self, sandbox_id: String, @@ -820,6 +845,7 @@ mod linux { policy: SandboxPolicyWire, ca_cert: Option>, ca_bundle: Option>, + provider_env_revision: u64, provider_env: std::collections::HashMap, ) -> Response { let mut state = lock(&self.state); @@ -829,6 +855,7 @@ mod linux { policy: policy.clone(), ca_cert: ca_cert.clone(), ca_bundle: ca_bundle.clone(), + provider_env_revision, provider_env: provider_env.clone(), }; if let RuntimeState::Running(process) = &*state { @@ -870,6 +897,7 @@ mod linux { spec, policy, resolved_identity, + provider_env_revision, provider_env, ca_file_paths, }; @@ -884,6 +912,27 @@ mod linux { Response::Started { process_id } } + fn update_provider_environment( + &self, + revision: u64, + provider_env: std::collections::HashMap, + ) -> Response { + let process = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return guest_error( + "invalid", + "agent process must be running before provider environment updates", + ); + }; + process.clone() + }; + let revision = process + .provider_credentials + .install_child_env_snapshot_if_newer(revision, provider_env); + Response::ProviderEnvironmentUpdated { revision } + } + fn wait(&self, process_id: &str) -> Response { let process = match self.running_process(process_id) { Ok(process) => process, @@ -1021,6 +1070,7 @@ mod linux { attached: AtomicBool, _netns: Option>, boundary_runtime: Arc, + provider_credentials: ProviderCredentialState, } struct ManagedProcessLaunch { @@ -1028,6 +1078,7 @@ mod linux { spec: AgentSpecWire, policy: openshell_core::policy::SandboxPolicy, resolved_identity: ResolvedProcessIdentity, + provider_env_revision: u64, provider_env: std::collections::HashMap, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, } @@ -1043,6 +1094,7 @@ mod linux { spec, policy, resolved_identity, + provider_env_revision, provider_env, ca_file_paths, } = launch; @@ -1051,6 +1103,10 @@ mod linux { } let boundary_runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); let entrypoint_pid = Arc::new(AtomicU32::new(0)); + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + provider_env_revision, + provider_env.clone(), + ); let mut spawned = runtime .block_on(spawn_workload( &spec.program, @@ -1067,7 +1123,7 @@ mod linux { ProcessEnforcementMode::Full, entrypoint_pid.clone(), None, - ProviderCredentialState::from_child_env_snapshot(0, provider_env.clone()), + provider_credentials.clone(), provider_env, ca_file_paths, AgentProposals::default(), @@ -1109,6 +1165,7 @@ mod linux { attached: AtomicBool::new(false), _netns: prepared.netns, boundary_runtime, + provider_credentials, }) } @@ -1897,6 +1954,7 @@ mod linux { policy.clone(), None, None, + 0, std::collections::HashMap::new(), ) }; @@ -1947,6 +2005,7 @@ mod linux { changed_policy, None, None, + 0, std::collections::HashMap::new(), ), Response::Error { kind, .. } if kind == "denied" @@ -2038,6 +2097,7 @@ mod linux { Some(nix::unistd::Uid::current().as_raw()), Some(nix::unistd::Gid::current().as_raw()), ), + provider_env_revision: 0, provider_env: std::collections::HashMap::new(), ca_file_paths: None, }, @@ -2069,6 +2129,7 @@ mod linux { policy: wire_policy.clone(), ca_cert: None, ca_bundle: None, + provider_env_revision: 0, provider_env: std::collections::HashMap::new(), }); @@ -2083,6 +2144,7 @@ mod linux { wire_policy, None, None, + 0, std::collections::HashMap::new(), ), Response::Started { @@ -2090,6 +2152,42 @@ mod linux { } ); + assert_eq!( + boundary.update_provider_environment( + 2, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "refreshed".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 2 } + ); + assert_eq!( + boundary.update_provider_environment( + 1, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "stale".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 2 } + ); + + let (cancelled_id, cancelled_session) = boundary + .start_exec(ExecSpecWire { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + env: Vec::new(), + workdir: None, + pty: false, + }) + .expect("start exec whose response is cancelled"); + boundary.cancel_exec_start(&cancelled_id, cancelled_session.process.clone()); + assert!( + !lock(&boundary.exec_handles).contains_key(&cancelled_id), + "a cancelled exec handoff must remove its process handle" + ); + let accept_boundary = boundary; let pending_accept = std::thread::spawn(move || accept_boundary.accept_network()); let deadline = std::time::Instant::now() + Duration::from_secs(5); @@ -2110,7 +2208,10 @@ mod linux { process.boundary_exec().exec( ExecSpecWire { program: "/bin/sh".to_string(), - args: vec!["-c".to_string(), "printf retained".to_string()], + args: vec![ + "-c".to_string(), + "printf %s \"$ROTATED_TOKEN\"".to_string(), + ], env: Vec::new(), workdir: None, pty: false, @@ -2123,7 +2224,7 @@ mod linux { process_runtime .block_on(session.stdout.read_to_string(&mut output)) .expect("read retained exec output"); - assert_eq!(output, "retained"); + assert_eq!(output, "refreshed"); assert!(matches!( process_runtime.block_on(session.process.wait()), Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(0)) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index e851dfb1f7..84e958e8cf 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -465,7 +465,7 @@ pub async fn run_sandbox( openshell_isolation_interface::remote::RemoteIsolationBackend::new( admitted_backend_name.clone(), ca_file_paths.clone(), - provider_env.clone(), + provider_credentials.clone(), ), ); let mut registry = openshell_isolation_interface::contract::BackendRegistry::new(); @@ -1056,13 +1056,14 @@ pub async fn run_sandbox( (exit_code, false) } }; + boundary_access.publish_main_exit(exit_code).await; if retain_access && let Some(marker) = main_exit_marker.as_deref() { persist_main_exit_marker(marker, exit_code) .into_diagnostic() .wrap_err("persist canonical-process completion marker")?; } if let (Some(endpoint), Some(id)) = (openshell_endpoint.as_deref(), sandbox_id.as_deref()) { - openshell_supervisor_process::delegated::report_main_process_completion( + openshell_supervisor_process::delegated::report_main_process_exit( endpoint, id, &instance_id, @@ -1070,7 +1071,15 @@ pub async fn run_sandbox( ) .await; } - boundary_access.finish_main(exit_code).await; + boundary_access.drain_main_terminal_delivery().await; + if let (Some(endpoint), Some(id)) = (openshell_endpoint.as_deref(), sandbox_id.as_deref()) { + openshell_supervisor_process::delegated::finalize_main_process_exit( + endpoint, + id, + &instance_id, + ) + .await; + } if retain_access { info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 7cd4fbf238..9467e138f0 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -49,13 +49,20 @@ impl BoundaryAccess { &self.instance_id } - /// Complete the control-side canonical-process attachment after the - /// boundary reports terminal status. - pub async fn finish_main(&self, exit_code: i32) { + /// Publish the canonical process's terminal status to attached clients. + pub async fn publish_main_exit(&self, exit_code: i32) { let Some(main_session) = self.main_session.as_ref() else { return; }; let _ = main_session.finish(exit_code, false).await; + } + + /// Release terminal delivery after the gateway acknowledges the exit, then + /// wait for attached clients to consume the terminal status. + pub async fn drain_main_terminal_delivery(&self) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; main_session.mark_terminal_reported(); main_session.wait_for_terminal_attachments().await; } @@ -493,8 +500,8 @@ fn ssh_proxy_url_for_policy( ) } -/// Report and finalize the canonical process exit from control mode. -pub async fn report_main_process_completion( +/// Report the canonical process exit from control mode until acknowledged. +pub async fn report_main_process_exit( endpoint: &str, sandbox_id: &str, instance_id: &str, @@ -518,7 +525,11 @@ pub async fn report_main_process_completion( } } } - delay = Duration::from_millis(250); +} + +/// Finalize canonical process terminal delivery until acknowledged. +pub async fn finalize_main_process_exit(endpoint: &str, sandbox_id: &str, instance_id: &str) { + let mut delay = Duration::from_millis(250); loop { match crate::supervisor_session::finalize_main_process_exit( endpoint, diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 346eac7bac..7f5dd9fcee 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -12,6 +12,8 @@ use crate::process::{ is_supervisor_only_env_var, }; use crate::sandbox; +#[cfg(unix)] +use libc; use miette::{IntoDiagnostic, Result}; use nix::pty::{Winsize, openpty}; use nix::unistd::setsid; @@ -166,27 +168,150 @@ pub async fn run_ssh_server( } }; + let mut consecutive_resource_errors = 0; + let mut consecutive_unknown_errors = 0; loop { - let (stream, _peer) = listener.accept().await.into_diagnostic()?; - let config = config.clone(); - let port_forward = port_forward.clone(); - let boundary_exec = boundary_exec.clone(); - let main_session = main_session.clone(); - - tokio::spawn(async move { - if let Err(err) = - handle_connection(stream, config, port_forward, boundary_exec, main_session).await - { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .message(format!("SSH connection failed: {err}")) - .build() - ); + match listener.accept().await { + Ok((stream, _peer)) => { + consecutive_resource_errors = 0; + consecutive_unknown_errors = 0; + let config = config.clone(); + let port_forward = port_forward.clone(); + let boundary_exec = boundary_exec.clone(); + let main_session = main_session.clone(); + + tokio::spawn(async move { + if let Err(err) = + handle_connection(stream, config, port_forward, boundary_exec, main_session) + .await + { + ocsf_emit!( + SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .message(format!("SSH connection failed: {err}")) + .build() + ); + } + }); } - }); + Err(error) => match classify_ssh_accept_error( + &error, + &mut consecutive_resource_errors, + &mut consecutive_unknown_errors, + ) { + SshAcceptAction::Terminal => { + return Err(error).into_diagnostic(); + } + SshAcceptAction::Retry { backoff, severity } => { + ocsf_emit!( + SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(severity) + .status(StatusId::Failure) + .message(format!( + "SSH accept error (retrying in {}ms): {error}", + backoff.as_millis() + )) + .build() + ); + tokio::time::sleep(backoff).await; + } + }, + } + } +} + +const MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS: u32 = 10; + +#[derive(Debug, PartialEq)] +enum SshAcceptAction { + Terminal, + Retry { + backoff: Duration, + severity: SeverityId, + }, +} + +fn classify_ssh_accept_error( + error: &std::io::Error, + consecutive_resource_errors: &mut u32, + consecutive_unknown_errors: &mut u32, +) -> SshAcceptAction { + #[cfg(unix)] + if matches!( + error.raw_os_error(), + Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK) + ) { + return SshAcceptAction::Terminal; + } + + #[cfg(unix)] + if matches!( + error.raw_os_error(), + Some( + libc::EMFILE + | libc::ENFILE + | libc::ENOBUFS + | libc::ENOMEM + | libc::ECONNABORTED + | libc::ECONNRESET + | libc::EINTR + | libc::ENETDOWN + | libc::EPROTO + | libc::ENOPROTOOPT + | libc::EHOSTDOWN + | libc::EHOSTUNREACH + | libc::EOPNOTSUPP + | libc::ENETUNREACH + | libc::ENOSR + | libc::ESOCKTNOSUPPORT + | libc::EPROTONOSUPPORT + | libc::ETIMEDOUT + ) + ) { + *consecutive_unknown_errors = 0; + let resource_pressure = matches!( + error.raw_os_error(), + Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM | libc::ENOSR) + ); + if resource_pressure { + *consecutive_resource_errors = consecutive_resource_errors.saturating_add(1); + let backoff_ms = 100_u64 + .saturating_mul(1_u64 << (*consecutive_resource_errors).min(7).saturating_sub(1)) + .min(5_000); + return SshAcceptAction::Retry { + backoff: Duration::from_millis(backoff_ms), + severity: SeverityId::Medium, + }; + } + *consecutive_resource_errors = 0; + return SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Low, + }; + } + + #[cfg(target_os = "linux")] + if error.raw_os_error() == Some(libc::ENONET) { + *consecutive_resource_errors = 0; + *consecutive_unknown_errors = 0; + return SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Low, + }; + } + + *consecutive_resource_errors = 0; + *consecutive_unknown_errors = consecutive_unknown_errors.saturating_add(1); + if *consecutive_unknown_errors >= MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS { + SshAcceptAction::Terminal + } else { + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Low, + } } } @@ -1748,6 +1873,53 @@ mod tests { use super::*; use std::process::Stdio; + #[cfg(unix)] + #[test] + fn transient_accept_errors_retry_with_bounded_backoff() { + let mut resource_errors = 0; + let mut unknown_errors = 0; + let aborted = std::io::Error::from_raw_os_error(libc::ECONNABORTED); + assert_eq!( + classify_ssh_accept_error(&aborted, &mut resource_errors, &mut unknown_errors), + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Low, + } + ); + + let exhausted = std::io::Error::from_raw_os_error(libc::EMFILE); + let first = + classify_ssh_accept_error(&exhausted, &mut resource_errors, &mut unknown_errors); + let second = + classify_ssh_accept_error(&exhausted, &mut resource_errors, &mut unknown_errors); + assert_eq!( + first, + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Medium, + } + ); + assert_eq!( + second, + SshAcceptAction::Retry { + backoff: Duration::from_millis(200), + severity: SeverityId::Medium, + } + ); + } + + #[cfg(unix)] + #[test] + fn invalid_listener_accept_error_is_terminal() { + let mut resource_errors = 0; + let mut unknown_errors = 0; + let error = std::io::Error::from_raw_os_error(libc::EBADF); + assert_eq!( + classify_ssh_accept_error(&error, &mut resource_errors, &mut unknown_errors), + SshAcceptAction::Terminal + ); + } + #[tokio::test] async fn main_attachment_accepts_declared_session_after_process_exit() { let main_session = MainSession::inert(); From 5d1a2c288ade1750cfc309810461a7cccee12c41 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 01:47:06 -0700 Subject: [PATCH 18/37] feat(isolation): expose control readiness listener Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/lib.rs | 82 +++++++++++++++++++++++++++- crates/openshell-sandbox/src/main.rs | 4 +- rfc/0012-isolation-backend/README.md | 7 +++ 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 84e958e8cf..1d42bd0951 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -94,6 +94,54 @@ async fn retain_remote_access_plane( } } +struct ControlReadiness { + task: tokio::task::JoinHandle<()>, + #[cfg(test)] + address: std::net::SocketAddr, +} + +impl ControlReadiness { + async fn start(port: u16) -> Result { + if port == 0 { + return Err(miette::miette!( + "control-mode readiness port must be nonzero" + )); + } + Self::start_at((std::net::Ipv4Addr::UNSPECIFIED, port).into()).await + } + + async fn start_at(address: std::net::SocketAddr) -> Result { + let listener = tokio::net::TcpListener::bind(address) + .await + .into_diagnostic() + .wrap_err_with(|| format!("bind control-mode readiness listener on {address}"))?; + #[cfg(test)] + let address = listener.local_addr().into_diagnostic()?; + let task = tokio::spawn(async move { + loop { + match listener.accept().await { + Ok((stream, _)) => drop(stream), + Err(error) => { + tracing::warn!(%error, "control-mode readiness accept failed; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + }); + Ok(Self { + task, + #[cfg(test)] + address, + }) + } +} + +impl Drop for ControlReadiness { + fn drop(&mut self) { + self.task.abort(); + } +} + #[cfg(unix)] async fn wait_for_control_shutdown_signal() { use tokio::signal::unix::{SignalKind, signal}; @@ -163,8 +211,8 @@ pub async fn run_sandbox( policy_rules: Option, policy_data: Option, ssh_socket_path: Option, - _health_check: bool, - _health_port: u16, + health_check: bool, + health_port: u16, inference_routes: Option, ocsf_enabled: Arc, network_enabled: bool, @@ -1010,6 +1058,11 @@ pub async fn run_sandbox( ) .await?; info!(backend = %backend_name, "Control-mode access plane started"); + let mut control_readiness = if health_check { + Some(ControlReadiness::start(health_port).await?) + } else { + None + }; let instance_id = boundary_access.instance_id().to_string(); let wait_agent = agent.clone(); let shutdown_requested = wait_for_control_shutdown_signal(); @@ -1056,6 +1109,9 @@ pub async fn run_sandbox( (exit_code, false) } }; + if !retain_access { + control_readiness.take(); + } boundary_access.publish_main_exit(exit_code).await; if retain_access && let Some(marker) = main_exit_marker.as_deref() { persist_main_exit_marker(marker, exit_code) @@ -1084,6 +1140,7 @@ pub async fn run_sandbox( info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; } + drop(control_readiness); drop(running); drop(boundary_access); exit_code @@ -4752,6 +4809,27 @@ fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String mod tests { use super::*; + #[tokio::test] + async fn control_readiness_exists_only_while_guard_is_live() { + let readiness = ControlReadiness::start_at("127.0.0.1:0".parse().unwrap()) + .await + .expect("start readiness listener"); + let address = readiness.address; + tokio::net::TcpStream::connect(address) + .await + .expect("running control accepts readiness probes"); + + drop(readiness); + tokio::task::yield_now().await; + let result = timeout( + Duration::from_secs(1), + tokio::net::TcpStream::connect(address), + ) + .await + .expect("closed readiness listener fails promptly"); + assert!(result.is_err()); + } + #[test] fn main_exit_marker_atomically_replaces_previous_value() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index d625190029..d7344199cc 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -185,11 +185,11 @@ struct Args { #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] inference_routes: Option, - /// Enable health check endpoint. + /// Expose control-mode TCP readiness after the boundary is running. #[arg(long)] health_check: bool, - /// Port for health check endpoint. + /// TCP readiness port used with `--health-check`. #[arg(long, default_value = "8080")] health_port: u16, diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md index 754f545397..b10b5b0cb5 100644 --- a/rfc/0012-isolation-backend/README.md +++ b/rfc/0012-isolation-backend/README.md @@ -89,6 +89,13 @@ Co-located deployments may keep the existing in-process backend and omit boundar `openshell-sandbox --mode=control` runs outside the untrusted execution environment. It receives the admitted backend name and a protected `TopologyDescriptor`, resolves the backend without fallback, and owns the gateway-facing access plane. It is the only mode that possesses gateway credentials or applies approved network policy. +Orchestrators may run control mode with `--health-check --health-port `. +The TCP listener becomes reachable only after `start_agent` has returned a +`RunningBoundary` and the gateway-facing access plane is established. Control +mode drops the listener when that boundary/access-plane lifetime ends, so a +Kubernetes `tcpSocket` readiness probe observes semantic control readiness +rather than mere process liveness. The listener carries no application data. + `openshell-sandbox --mode=boundary --boundary-config ` runs inside, or immediately adjacent to, the execution environment. The driver supplies the config over a protected filesystem channel. It names one boundary, one private listener, one per-boundary bootstrap credential, and the workload identity. The identity is either a platform-resolved numeric UID/GID pair or an OCI `Config.User` declaration that boundary mode resolves against the workload filesystem. Boundary mode authenticates and scopes every request to that boundary. It never accepts policy or identity claims from workload code and it cannot authorize an operation independently of control mode. The shared protocol is versioned independently of a driver's resource descriptor. It carries `attach`, `confirm`, `start_agent`, agent and exec `wait`/`signal`/`terminate`, PTY resize, loopback-only forwarding, and attributed egress streams. Drivers may use a private Unix socket, TLS-authenticated TCP, a Unix endpoint mapped to guest vsock, or host `AF_VSOCK`; adding a transport does not change lifecycle or operation semantics. A TCP transport that crosses a shared or operator-managed network must authenticate the server name against a driver-provisioned trust root and encrypt every control and stream request. Network isolation alone is not a confidentiality boundary. Secrets are delivered in protected files, redacted from debug output, and never placed in workload-visible environment or process arguments. When a bind-mounted file's host owner may match the workload UID, the driver requires boundary mode to re-own it as root-only before the first workload instruction. Unix socket inodes permit cross-UID control within their private driver-owned directory; the per-boundary bootstrap credential authenticates every request. From 129c8238d6c8fd61ef22c5e5ea6dee502c4f5ab7 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 02:09:26 -0700 Subject: [PATCH 19/37] fix(isolation): harden delegated boundary transport Signed-off-by: Drew Newberry --- Cargo.lock | 1 + architecture/sandbox.md | 9 + .../openshell-isolation-interface/Cargo.toml | 1 + .../src/remote.rs | 22 +++ .../openshell-sandbox/src/boundary_server.rs | 145 ++++++++++++-- crates/openshell-sandbox/src/lib.rs | 60 +++++- .../src/delegated.rs | 2 +- .../src/main_session.rs | 51 +++++ .../openshell-supervisor-process/src/ssh.rs | 187 ++++++++++++++++-- 9 files changed, 431 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f94f979cab..cd0bdeb494 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4183,6 +4183,7 @@ dependencies = [ "rustls-pemfile", "serde", "serde_json", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tokio-rustls 0.26.4", diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 1d70757cf8..8dbd0916d2 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -136,6 +136,15 @@ generation-pinned authorization form the transparent TCP security boundary. Docker and Podman do not currently advertise usable IPv6 egress for this substrate, so AAAA queries return NOERROR/NODATA and IPv6 DNS remains fenced. +Network-rule ownership depends on supervisor placement. In legacy combined +mode, `openshell-sandbox` creates the workload network namespace and owns its +`ip`, iptables, and nftables setup and cleanup. In delegated control/boundary +mode, the compute driver provisions the protected network substrate before +control attaches; boundary mode owns only boundary-local observation and the +listener handed to it. Control mode evaluates policy but must not mutate a +driver-owned namespace or assume that legacy `ip`/nft cleanup applies. Drivers +must tear down their own rules and namespace resources with the workload. + Provider credential placeholders are resolved through the live provider state for each HTTP request, after destination and L7 policy admission. A static credential resolves only when the request host, port, and path match an endpoint diff --git a/crates/openshell-isolation-interface/Cargo.toml b/crates/openshell-isolation-interface/Cargo.toml index 43e91a6986..401eb280cc 100644 --- a/crates/openshell-isolation-interface/Cargo.toml +++ b/crates/openshell-isolation-interface/Cargo.toml @@ -20,6 +20,7 @@ tokio = { workspace = true } rustls = { workspace = true } rustls-pemfile = { workspace = true } tokio-rustls = { workspace = true } +socket2 = { workspace = true } [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index caffcb1df1..58465a918f 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -891,6 +891,7 @@ impl BoundaryClient { "connect to boundary TCP endpoint {address}: {error}" )) })?; + enable_boundary_tcp_keepalive(&stream); Ok(Box::new(stream)) } BoundaryTransport::TlsTcp { @@ -905,6 +906,7 @@ impl BoundaryClient { "connect to boundary TLS endpoint {address}: {error}" )) })?; + enable_boundary_tcp_keepalive(&stream); let server_name = rustls::pki_types::ServerName::try_from(server_name.clone()) .map_err(|error| { BackendError::Descriptor(format!( @@ -932,6 +934,13 @@ impl BoundaryClient { } } +fn enable_boundary_tcp_keepalive(stream: &tokio::net::TcpStream) { + let keepalive = socket2::TcpKeepalive::new() + .with_time(Duration::from_secs(30)) + .with_interval(Duration::from_secs(10)); + let _ = socket2::SockRef::from(stream).set_tcp_keepalive(&keepalive); +} + #[cfg(target_os = "linux")] fn connect_host_vsock( guest_cid: u32, @@ -1035,6 +1044,19 @@ mod tests { use super::*; + #[tokio::test] + async fn boundary_tcp_connections_enable_keepalive() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let connected = tokio::spawn(async move { tokio::net::TcpStream::connect(address).await }); + let (_server, _) = listener.accept().await.unwrap(); + let client = connected.await.unwrap().unwrap(); + + enable_boundary_tcp_keepalive(&client); + + assert!(socket2::SockRef::from(&client).keepalive().unwrap()); + } + struct TestCertificate { ca_pem: String, server_config: Arc, diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index dca358ee98..a4ecb84c13 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -21,7 +21,7 @@ mod linux { use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd as _, OwnedFd}; use std::os::unix::ffi::OsStrExt as _; use std::os::unix::fs::PermissionsExt as _; - use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; + use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Condvar, Mutex}; use std::time::Duration; @@ -49,10 +49,30 @@ mod linux { BoundaryConfig, BoundaryListener as BoundaryListenerConfig, ExecSpecWire, ExitStatusWire, Request, RequestEnvelope, Response, ResponseEnvelope, STREAM_EXIT, STREAM_STDERR, STREAM_STDIN, STREAM_STDIN_CLOSED, STREAM_STDOUT, SandboxPolicyWire, SignalWire, - read_frame, read_stream_frame, validate_resource_claims, write_frame, write_stream_frame, + encode_frame, read_frame, read_stream_frame, validate_resource_claims, write_frame, + write_stream_frame, }; const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30); + const OUTPUT_DRAIN_TIMEOUT: Duration = Duration::from_secs(2); + const MAX_CONTROL_CONNECTIONS: usize = 128; + + struct ControlConnectionSlot(Arc); + + impl Drop for ControlConnectionSlot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } + } + + fn acquire_control_connection_slot(active: &Arc) -> Option { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < MAX_CONTROL_CONNECTIONS).then_some(current + 1) + }) + .ok() + .map(|_| ControlConnectionSlot(active.clone())) + } static BOUNDARY_TERMINATION_REQUESTED: AtomicBool = AtomicBool::new(false); extern "C" fn request_boundary_termination(_signal: libc::c_int) { @@ -235,6 +255,7 @@ mod linux { fn serve(config: &BoundaryListenerConfig, runtime: Arc) -> Result<(), String> { let listener = ControlListener::bind(config) .map_err(|error| format!("bind boundary control listener: {error}"))?; + let active_connections = Arc::new(AtomicUsize::new(0)); tracing::info!(?config, "Boundary control listener ready"); loop { if BOUNDARY_TERMINATION_REQUESTED.load(Ordering::Acquire) { @@ -243,8 +264,16 @@ mod linux { } match listener.accept() { Ok(stream) => { + let Some(slot) = acquire_control_connection_slot(&active_connections) else { + tracing::warn!( + limit = MAX_CONTROL_CONNECTIONS, + "Boundary control connection limit reached" + ); + continue; + }; let runtime = runtime.clone(); std::thread::spawn(move || { + let _slot = slot; let stream = match stream.establish(&runtime.process_runtime) { Ok(stream) => stream, Err(error) => { @@ -357,17 +386,29 @@ mod linux { return Ok(()); } Request::AcceptNetwork => { - let (mut target, identity) = runtime.accept_network()?; - write_frame( - &mut stream, - &ResponseEnvelope { - request_id: request.request_id, - response: Response::NetworkConnected { identity }, - }, - ) - .map_err(|error| format!("write network mediation response: {error}"))?; + let (listener, process) = runtime.network_accept_context()?; + let request_id = request.request_id; runtime.process_runtime.block_on(async move { let mut stream = stream.into_tokio()?; + let (mut target, workload_addr) = + accept_network_while_control_connected(&listener, &mut stream).await?; + let proxy_addr = target + .local_addr() + .map_err(|error| format!("read boundary proxy address: {error}"))?; + let identity = BinaryIdentityWire::from( + process + .identity_resolver() + .resolve_connection(workload_addr, proxy_addr), + ); + let response = encode_frame(&ResponseEnvelope { + request_id, + response: Response::NetworkConnected { identity }, + }) + .map_err(|error| format!("encode network mediation response: {error}"))?; + stream + .write_all(&response) + .await + .map_err(|error| format!("write network mediation response: {error}"))?; tokio::io::copy_bidirectional(&mut stream, &mut target) .await .map_err(|error| { @@ -387,6 +428,24 @@ mod linux { Ok(()) } + async fn accept_network_while_control_connected( + listener: &tokio::net::TcpListener, + control: &mut openshell_isolation_interface::contract::BoundaryDuplexStream, + ) -> Result<(tokio::net::TcpStream, std::net::SocketAddr), String> { + let mut unexpected_control_data = [0_u8; 1]; + tokio::select! { + biased; + control_result = control.read(&mut unexpected_control_data) => match control_result { + Ok(0) => Err("AcceptNetwork cancelled when control disconnected".to_string()), + Ok(_) => Err("unexpected data after AcceptNetwork request".to_string()), + Err(error) => Err(format!("monitor AcceptNetwork control stream: {error}")), + }, + accepted = listener.accept() => accepted.map_err(|error| { + format!("accept boundary proxy connection: {error}") + }), + } + } + struct BoundaryRuntime { config: BoundaryConfig, process_runtime: tokio::runtime::Handle, @@ -496,15 +555,9 @@ mod linux { .wait() .await .map_err(|error| format!("wait for boundary exec: {error}"))?; - stdout_task - .await - .map_err(|error| format!("join boundary exec stdout: {error}"))? - .map_err(|error| format!("stream boundary exec stdout: {error}"))?; + drain_exec_output(stdout_task, "stdout").await?; if let Some(stderr_task) = stderr_task { - stderr_task - .await - .map_err(|error| format!("join boundary exec stderr: {error}"))? - .map_err(|error| format!("stream boundary exec stderr: {error}"))?; + drain_exec_output(stderr_task, "stderr").await?; } let exit = serde_json::to_vec(&ExitStatusWire::from(status)) .map_err(|error| format!("encode boundary exec exit: {error}"))?; @@ -517,6 +570,21 @@ mod linux { Ok(()) } + async fn drain_exec_output( + mut task: tokio::task::JoinHandle>, + name: &str, + ) -> Result<(), String> { + if let Ok(result) = tokio::time::timeout(OUTPUT_DRAIN_TIMEOUT, &mut task).await { + result + .map_err(|error| format!("join boundary exec {name}: {error}"))? + .map_err(|error| format!("stream boundary exec {name}: {error}")) + } else { + task.abort(); + tracing::warn!(stream = name, "boundary exec output drain timed out"); + Ok(()) + } + } + async fn pump_exec_output( mut output: openshell_isolation_interface::contract::BoundaryOutput, channel: u8, @@ -718,7 +786,9 @@ mod linux { .map_err(|error| error.to_string()) } - fn accept_network(&self) -> Result<(tokio::net::TcpStream, BinaryIdentityWire), String> { + fn network_accept_context( + &self, + ) -> Result<(Arc, Arc), String> { let process = loop { let running = { let state = lock(&self.state); @@ -735,6 +805,12 @@ mod linux { let listener = process .network_listener() .ok_or_else(|| "network mediation requested for a non-proxy policy".to_string())?; + Ok((listener, process)) + } + + #[cfg(test)] + fn accept_network(&self) -> Result<(tokio::net::TcpStream, BinaryIdentityWire), String> { + let (listener, process) = self.network_accept_context()?; let (stream, workload_addr) = self .process_runtime .block_on(listener.accept()) @@ -1759,6 +1835,35 @@ mod linux { assert!(!constant_time_eq(b"same", b"sam")); } + #[test] + fn control_connection_slots_bound_unauthenticated_threads() { + let active = Arc::new(AtomicUsize::new(MAX_CONTROL_CONNECTIONS - 1)); + let slot = acquire_control_connection_slot(&active).expect("last available slot"); + assert!(acquire_control_connection_slot(&active).is_none()); + drop(slot); + assert_eq!(active.load(Ordering::Acquire), MAX_CONTROL_CONNECTIONS - 1); + } + + #[tokio::test] + async fn pending_network_accept_cancels_when_control_disconnects() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind workload listener"); + let (control, peer) = tokio::io::duplex(64); + drop(peer); + let mut control: openshell_isolation_interface::contract::BoundaryDuplexStream = + Box::new(control); + + let error = tokio::time::timeout( + Duration::from_secs(1), + accept_network_while_control_connected(&listener, &mut control), + ) + .await + .expect("disconnect cancels accept promptly") + .expect_err("disconnected control cannot retain AcceptNetwork"); + assert!(error.contains("control disconnected")); + } + #[test] fn unix_listener_allows_authenticated_cross_uid_control() { use std::os::unix::fs::PermissionsExt as _; diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 1d42bd0951..9b4e5eaf20 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -94,6 +94,18 @@ async fn retain_remote_access_plane( } } +async fn completion_phase_or_shutdown(phase: F, mut shutdown: Pin<&mut S>) -> bool +where + F: Future, + S: Future + ?Sized, +{ + tokio::pin!(phase); + tokio::select! { + () = &mut phase => false, + () = &mut shutdown => true, + } +} + struct ControlReadiness { task: tokio::task::JoinHandle<()>, #[cfg(test)] @@ -1081,7 +1093,7 @@ pub async fn run_sandbox( }) .map_err(|error| miette::miette!(error.to_string())) }; - let (exit_code, retain_access) = tokio::select! { + let (exit_code, mut retain_access) = tokio::select! { result = wait => (result?, true), () = &mut proxy_exited => { let _ = agent.terminate().await; @@ -1113,28 +1125,47 @@ pub async fn run_sandbox( control_readiness.take(); } boundary_access.publish_main_exit(exit_code).await; + // `shutdown_requested` has already completed when shutdown won the + // lifecycle select above and must not be polled again. + let mut completion_cancelled = !retain_access; if retain_access && let Some(marker) = main_exit_marker.as_deref() { persist_main_exit_marker(marker, exit_code) .into_diagnostic() .wrap_err("persist canonical-process completion marker")?; } - if let (Some(endpoint), Some(id)) = (openshell_endpoint.as_deref(), sandbox_id.as_deref()) { - openshell_supervisor_process::delegated::report_main_process_exit( + if !completion_cancelled + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + { + let report = openshell_supervisor_process::delegated::report_main_process_exit( endpoint, id, &instance_id, exit_code, - ) - .await; + ); + completion_cancelled = + completion_phase_or_shutdown(report, shutdown_requested.as_mut()).await; + } + if !completion_cancelled { + let drain = boundary_access.drain_main_terminal_delivery(); + completion_cancelled = + completion_phase_or_shutdown(drain, shutdown_requested.as_mut()).await; } - boundary_access.drain_main_terminal_delivery().await; - if let (Some(endpoint), Some(id)) = (openshell_endpoint.as_deref(), sandbox_id.as_deref()) { - openshell_supervisor_process::delegated::finalize_main_process_exit( + if !completion_cancelled + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + { + let finalize = openshell_supervisor_process::delegated::finalize_main_process_exit( endpoint, id, &instance_id, - ) - .await; + ); + completion_cancelled = + completion_phase_or_shutdown(finalize, shutdown_requested.as_mut()).await; + } + if completion_cancelled { + retain_access = false; + control_readiness.take(); } if retain_access { info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); @@ -4868,6 +4899,15 @@ mod tests { .expect("clean teardown"); } + #[tokio::test] + async fn completion_retry_phase_is_cancelled_by_shutdown() { + let mut shutdown = Box::pin(std::future::ready(())); + assert!( + completion_phase_or_shutdown(std::future::pending(), shutdown.as_mut()).await, + "shutdown must cancel an indefinitely retrying completion phase" + ); + } + #[test] fn transparent_tcp_capability_requires_exact_driver_marker() { let required = openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY; diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 9467e138f0..db0ea9b56e 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -54,7 +54,7 @@ impl BoundaryAccess { let Some(main_session) = self.main_session.as_ref() else { return; }; - let _ = main_session.finish(exit_code, false).await; + let _ = main_session.finish_remote(exit_code, false).await; } /// Release terminal delivery after the gateway acknowledges the exit, then diff --git a/crates/openshell-supervisor-process/src/main_session.rs b/crates/openshell-supervisor-process/src/main_session.rs index e91bf9d0f2..2aba0c48c1 100644 --- a/crates/openshell-supervisor-process/src/main_session.rs +++ b/crates/openshell-supervisor-process/src/main_session.rs @@ -200,6 +200,7 @@ pub struct MainSession { } impl MainSession { + const REMOTE_OUTPUT_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); #[cfg(test)] pub fn inert() -> Arc { let (input, _input_rx) = tokio::sync::mpsc::channel(64); @@ -465,10 +466,39 @@ impl MainSession { /// /// Returns whether terminal delivery must complete before shutdown. pub async fn finish(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.wait_for_output_readers().await; + self.complete_finish(exit_code, attachment_expected) + } + + /// Finish a remotely owned process without allowing descendants that keep + /// inherited output descriptors open to block terminal publication forever. + pub(crate) async fn finish_remote(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.finish_remote_with_timeout( + exit_code, + attachment_expected, + Self::REMOTE_OUTPUT_DRAIN_TIMEOUT, + ) + .await + } + + async fn finish_remote_with_timeout( + &self, + exit_code: i32, + attachment_expected: bool, + timeout: std::time::Duration, + ) -> bool { + let _ = tokio::time::timeout(timeout, self.wait_for_output_readers()).await; + self.complete_finish(exit_code, attachment_expected) + } + + async fn wait_for_output_readers(&self) { let notified = self.readers_done.notified(); if self.readers_remaining.load(Ordering::Acquire) != 0 { notified.await; } + } + + fn complete_finish(&self, exit_code: i32, attachment_expected: bool) -> bool { let delivery_pending = { let mut state = self .terminal_attachments @@ -817,6 +847,27 @@ mod tests { .expect("closing the attachment should wake the waiter"); } + #[tokio::test] + async fn remote_finish_bounds_output_drain_before_publishing_exit() { + let mut session = MainSession::inert(); + Arc::get_mut(&mut session) + .expect("sole test session reference") + .readers_remaining = AtomicUsize::new(1); + let mut output = session.subscribe(); + + session + .finish_remote_with_timeout(19, false, std::time::Duration::from_millis(10)) + .await; + + assert!(matches!( + output + .recv() + .await + .expect("terminal status after bounded drain"), + MainOutput::Exit(19) + )); + } + #[tokio::test] async fn declared_attachment_waits_for_connection_then_natural_close() { let session = MainSession::inert(); diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 7f5dd9fcee..e82e176942 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -515,9 +515,8 @@ impl russh::server::Handler for SshHandler { return Ok(()); } - // Only allow forwarding to loopback destinations to prevent the - // sandbox SSH server from being used as a generic proxy. - if !is_loopback_host(host_to_connect) { + let target = direct_tcpip_target(host_to_connect, port_to_connect); + if target.is_none() { ocsf_emit!(SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Refuse) .action(ActionId::Denied) @@ -536,19 +535,8 @@ impl russh::server::Handler for SshHandler { let host = host_to_connect.to_string(); // SSH protocol port is bounded by u32 but only u16 is meaningful; // saturate as a guard for malformed clients. - let port = u16::try_from(port_to_connect).unwrap_or(u16::MAX); - - // Build the loopback target up front. The host already passed - // `is_loopback_host`, and `LoopbackTarget::new` re-validates the parsed - // address (defense in depth) before the connect. - let Some(target) = loopback_ip(&host).and_then(|ip| { - openshell_isolation_interface::contract::LoopbackTarget::new(ip, port).ok() - }) else { - reply - .reject(ChannelOpenFailure::AdministrativelyProhibited) - .await; - return Ok(()); - }; + let port = u16::try_from(port_to_connect).expect("port range checked above"); + let target = target.expect("loopback target checked above"); let port_forward = self.port_forward.clone(); reply.accept().await; @@ -1863,6 +1851,18 @@ fn loopback_ip(host: &str) -> Option { host.parse().ok() } +fn direct_tcpip_target( + host: &str, + port: u32, +) -> Option { + if !is_loopback_host(host) { + return None; + } + let port = u16::try_from(port).ok()?; + let ip = loopback_ip(host)?; + openshell_isolation_interface::contract::LoopbackTarget::new(ip, port).ok() +} + #[cfg(test)] #[allow( clippy::doc_markdown, @@ -1873,6 +1873,100 @@ mod tests { use super::*; use std::process::Stdio; + struct AcceptAnyServerKey; + + impl russh::client::Handler for AcceptAnyServerKey { + type Error = russh::Error; + + async fn check_server_key( + &mut self, + _server_public_key: &russh::keys::PublicKey, + ) -> Result { + Ok(true) + } + } + + struct TestPortForward; + + #[async_trait::async_trait] + impl openshell_isolation_interface::contract::BoundaryPortForward for TestPortForward { + async fn connect( + &self, + target: openshell_isolation_interface::contract::LoopbackTarget, + ) -> std::result::Result< + openshell_isolation_interface::contract::BoundaryDuplexStream, + openshell_isolation_interface::contract::BackendError, + > { + let stream = tokio::net::TcpStream::connect((target.host(), target.port())) + .await + .map_err(|error| { + openshell_isolation_interface::contract::BackendError::Process( + error.to_string(), + ) + })?; + Ok(Box::new(stream)) + } + } + + struct RejectingExec; + + #[async_trait::async_trait] + impl openshell_isolation_interface::contract::BoundaryExec for RejectingExec { + async fn exec( + &self, + _spec: openshell_isolation_interface::contract::ExecSpec, + ) -> std::result::Result< + openshell_isolation_interface::contract::ExecSession, + openshell_isolation_interface::contract::BackendError, + > { + Err( + openshell_isolation_interface::contract::BackendError::Unsupported( + "exec is not used by direct-tcpip tests".into(), + ), + ) + } + } + + async fn authenticated_test_client() -> russh::client::Handle { + let host_key = { + let mut rng = rand::rng(); + PrivateKey::random(&mut rng, Algorithm::Ed25519).expect("host key") + }; + let mut server_config = russh::server::Config { + auth_rejection_time: Duration::from_millis(1), + ..Default::default() + }; + server_config.keys.push(host_key); + + let handler = SshHandler::new( + Arc::new(TestPortForward), + Arc::new(RejectingExec), + Some(MainSession::inert()), + ); + let (server_stream, client_stream) = tokio::io::duplex(64 * 1024); + tokio::spawn(async move { + if let Ok(session) = + russh::server::run_stream(Arc::new(server_config), server_stream, handler).await + { + let _ = session.await; + } + }); + + let mut client = russh::client::connect_stream( + Arc::new(russh::client::Config::default()), + client_stream, + AcceptAnyServerKey, + ) + .await + .expect("SSH handshake should complete over the duplex"); + let auth = client + .authenticate_none("sandbox") + .await + .expect("auth_none should not error"); + assert!(matches!(auth, russh::client::AuthResult::Success)); + client + } + #[cfg(unix)] #[test] fn transient_accept_errors_retry_with_bounded_backoff() { @@ -1920,6 +2014,67 @@ mod tests { ); } + #[test] + fn direct_tcpip_target_rejects_non_loopback_and_out_of_range_ports() { + assert!(direct_tcpip_target("10.0.0.1", 80).is_none()); + assert!(direct_tcpip_target("127.0.0.1", 65_537).is_none()); + } + + #[test] + fn direct_tcpip_target_accepts_loopback_destinations() { + let target = direct_tcpip_target("localhost", 8_080).expect("loopback target"); + assert_eq!( + target.host(), + std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST) + ); + assert_eq!(target.port(), 8_080); + } + + #[tokio::test] + async fn direct_tcpip_handler_rejects_invalid_destinations() { + for (host, port) in [("10.0.0.1", 80), ("127.0.0.1", 65_537)] { + let client = authenticated_test_client().await; + let error = client + .channel_open_direct_tcpip(host, port, "127.0.0.1", 0) + .await + .expect_err("invalid forwarding destination must be refused"); + assert!(matches!( + error, + russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) + )); + } + } + + #[tokio::test] + async fn direct_tcpip_handler_relays_loopback_bytes() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback echo listener"); + let port = listener.local_addr().expect("listener address").port(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept forwarded stream"); + let mut payload = [0_u8; 4]; + socket.read_exact(&mut payload).await.expect("read payload"); + socket.write_all(&payload).await.expect("echo payload"); + }); + + let client = authenticated_test_client().await; + let channel = client + .channel_open_direct_tcpip("127.0.0.1", u32::from(port), "127.0.0.1", 0) + .await + .expect("loopback forwarding must be allowed"); + let mut stream = channel.into_stream(); + stream.write_all(b"ping").await.expect("write channel"); + let mut echoed = [0_u8; 4]; + tokio::time::timeout(Duration::from_secs(5), stream.read_exact(&mut echoed)) + .await + .expect("forwarded response timeout") + .expect("read channel"); + assert_eq!(&echoed, b"ping"); + } + #[tokio::test] async fn main_attachment_accepts_declared_session_after_process_exit() { let main_session = MainSession::inert(); From a2b573194f2c118de8415fda62a388800be61553 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 02:14:39 -0700 Subject: [PATCH 20/37] fix(isolation): support IPv6 control readiness Signed-off-by: Drew Newberry --- architecture/sandbox.md | 4 +++ crates/openshell-core/src/sandbox_env.rs | 3 ++ crates/openshell-sandbox/src/lib.rs | 35 ++++++++++++++++++------ crates/openshell-sandbox/src/main.rs | 10 +++++++ rfc/0012-isolation-backend/README.md | 4 +++ 5 files changed, 48 insertions(+), 8 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 8dbd0916d2..f498e5d552 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -36,6 +36,10 @@ The shared remote backend authenticates each request and rejects a resource-clai mismatch. Driver crates do not appear in generic supervisor lifecycle, network, SSH, or session code. +Control mode can expose semantic TCP readiness after the boundary reaches +`Running`. The bind IP is explicit and address-family neutral; Kubernetes +drivers inject the pod IP so IPv4 and IPv6 probes reach the same listener. + For cross-UID Unix-socket placements, the driver-owned parent directory limits socket reachability and the bootstrap credential authenticates every request. Boundary mode can re-own a bind-mounted config as root-only before starting the diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 1feddc743d..9f51301759 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -26,6 +26,9 @@ pub const SSH_SOCKET_PATH: &str = "OPENSHELL_SSH_SOCKET_PATH"; /// Log level for the sandbox supervisor (e.g. `"debug"`, `"info"`, `"warn"`). pub const LOG_LEVEL: &str = "OPENSHELL_LOG_LEVEL"; +/// IP address for the control-mode semantic readiness listener. +pub const HEALTH_BIND_IP: &str = "OPENSHELL_HEALTH_BIND_IP"; + /// Versioned specification for the exact canonical main process. /// /// Most drivers use JSON directly. Transports that cannot preserve spaces in diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 9b4e5eaf20..c30c0b4273 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -113,13 +113,8 @@ struct ControlReadiness { } impl ControlReadiness { - async fn start(port: u16) -> Result { - if port == 0 { - return Err(miette::miette!( - "control-mode readiness port must be nonzero" - )); - } - Self::start_at((std::net::Ipv4Addr::UNSPECIFIED, port).into()).await + async fn start(bind_ip: std::net::IpAddr, port: u16) -> Result { + Self::start_at(control_readiness_address(bind_ip, port)?).await } async fn start_at(address: std::net::SocketAddr) -> Result { @@ -148,6 +143,15 @@ impl ControlReadiness { } } +fn control_readiness_address(bind_ip: std::net::IpAddr, port: u16) -> Result { + if port == 0 { + return Err(miette::miette!( + "control-mode readiness port must be nonzero" + )); + } + Ok(std::net::SocketAddr::new(bind_ip, port)) +} + impl Drop for ControlReadiness { fn drop(&mut self) { self.task.abort(); @@ -225,6 +229,7 @@ pub async fn run_sandbox( ssh_socket_path: Option, health_check: bool, health_port: u16, + health_bind_ip: std::net::IpAddr, inference_routes: Option, ocsf_enabled: Arc, network_enabled: bool, @@ -1071,7 +1076,7 @@ pub async fn run_sandbox( .await?; info!(backend = %backend_name, "Control-mode access plane started"); let mut control_readiness = if health_check { - Some(ControlReadiness::start(health_port).await?) + Some(ControlReadiness::start(health_bind_ip, health_port).await?) } else { None }; @@ -4861,6 +4866,20 @@ mod tests { assert!(result.is_err()); } + #[test] + fn control_readiness_preserves_explicit_ipv6_bind_address() { + let address = control_readiness_address("2001:db8::17".parse().unwrap(), 8_080) + .expect("valid IPv6 readiness address"); + assert_eq!(address, "[2001:db8::17]:8080".parse().unwrap()); + } + + #[test] + fn control_readiness_rejects_zero_port() { + let error = control_readiness_address(std::net::Ipv6Addr::UNSPECIFIED.into(), 0) + .expect_err("zero readiness port must be rejected"); + assert!(error.to_string().contains("must be nonzero")); + } + #[test] fn main_exit_marker_atomically_replaces_previous_value() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index d7344199cc..d0b2484179 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -193,6 +193,15 @@ struct Args { #[arg(long, default_value = "8080")] health_port: u16, + /// IP address for the control-mode TCP readiness listener. Kubernetes + /// IPv6 pods should inject `status.podIP` through `OPENSHELL_HEALTH_BIND_IP`. + #[arg( + long, + default_value = "0.0.0.0", + env = openshell_core::sandbox_env::HEALTH_BIND_IP + )] + health_bind_ip: std::net::IpAddr, + /// Supervisor role to run. Drivers use `control` outside the workload and /// `boundary` inside it. Legacy comma-separated leaf modes remain accepted. /// @@ -837,6 +846,7 @@ fn main() -> Result<()> { args.ssh_socket_path, args.health_check, args.health_port, + args.health_bind_ip, args.inference_routes, ocsf_enabled, args.mode.network, diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md index b10b5b0cb5..48e2ae3da1 100644 --- a/rfc/0012-isolation-backend/README.md +++ b/rfc/0012-isolation-backend/README.md @@ -90,6 +90,10 @@ Co-located deployments may keep the existing in-process backend and omit boundar `openshell-sandbox --mode=control` runs outside the untrusted execution environment. It receives the admitted backend name and a protected `TopologyDescriptor`, resolves the backend without fallback, and owns the gateway-facing access plane. It is the only mode that possesses gateway credentials or applies approved network policy. Orchestrators may run control mode with `--health-check --health-port `. +The listener defaults to `0.0.0.0`; `--health-bind-ip ` (or +`OPENSHELL_HEALTH_BIND_IP`) selects an explicit IPv4 or IPv6 address. Kubernetes +drivers should populate the environment variable from the Downward API +`status.podIP`, which keeps the probe address family aligned with the pod. The TCP listener becomes reachable only after `start_agent` has returned a `RunningBoundary` and the gateway-facing access plane is established. Control mode drops the listener when that boundary/access-plane lifetime ends, so a From b2da4a696b18188776f6874047b9a6439a079cf3 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 02:42:30 -0700 Subject: [PATCH 21/37] fix(network): preserve boundary endpoint metadata Signed-off-by: Drew Newberry --- .../openshell-supervisor-network/src/proxy.rs | 64 +++++++++++++++++-- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index b65ea8f908..0ffb64aeda 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -3055,13 +3055,13 @@ fn authorize_supplied_identity( ancestors: identity.ancestors.clone(), cmdline_paths: identity.cmdline_paths.clone(), }; - match engine.evaluate_network_action_with_generation(&input) { - Ok((action, generation)) => EgressDecision { + match engine.authorize_egress(&input) { + Ok(authorization) => EgressDecision { intent, - action, - policy_generation: generation, + action: authorization.action.clone(), + policy_generation: authorization.generation, identity: ProcessIdentityEvidence::Available, - endpoint: EndpointDecision::default(), + endpoint: EndpointDecision::from_authorization(&authorization), binary: Some(identity.binary_path.clone()), binary_pid: None, ancestors: identity.ancestors.clone(), @@ -6685,6 +6685,60 @@ mod tests { use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; + #[test] + fn supplied_identity_preserves_authorized_endpoint_metadata() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + inspected: + name: inspected + endpoints: + - host: api.example.com + port: 443 + protocol: rest + enforcement: enforce + request_body_credential_rewrite: true + allowed_ips: ["192.0.2.0/24"] + rules: + - allow: { method: GET, path: /allowed } + binaries: + - path: /usr/bin/python3 +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .expect("load policy"); + let identity = Ok(ContractBinaryIdentity { + binary_path: PathBuf::from("/usr/bin/python3"), + binary_digest: Some("00".repeat(32).parse().expect("digest")), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }); + + let mut decision = authorize_supplied_identity( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + &identity, + ); + + assert_eq!(query_allowed_ips(&decision), ["192.0.2.0/24"]); + hydrate_l7_route(&mut decision); + let route = decision + .endpoint + .l7_route + .expect("supplied identity must retain L7 metadata"); + assert_eq!(route.configs.len(), 1); + assert!(route.configs[0].config.request_body_credential_rewrite); + } + struct FailedMediationSource; #[tokio::test] From 6c0b3e62d9e636c7a51b1792209008aebf89a60e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 03:01:09 -0700 Subject: [PATCH 22/37] feat(supervisor): support durable interception CA material Signed-off-by: Drew Newberry --- Cargo.lock | 2 + Cargo.toml | 2 +- crates/openshell-core/src/sandbox_env.rs | 11 +++ crates/openshell-sandbox/src/lib.rs | 20 ++++- .../src/l7/tls.rs | 90 ++++++++++++++++++- .../openshell-supervisor-network/src/run.rs | 29 +++++- 6 files changed, 148 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd0bdeb494..2a614f6015 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5579,6 +5579,7 @@ dependencies = [ "ring", "rustls-pki-types", "time", + "x509-parser", "yasna", ] @@ -8665,6 +8666,7 @@ dependencies = [ "lazy_static", "nom", "oid-registry", + "ring", "rusticata-macros", "thiserror 1.0.69", "time", diff --git a/Cargo.toml b/Cargo.toml index 6936439a3b..16845307a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ http-body-util = "0.1" tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } rustls = { version = "0.23", default-features = false, features = ["std", "logging", "tls12", "ring"] } rustls-pemfile = "2" -rcgen = { version = "0.13", features = ["crypto", "pem"] } +rcgen = { version = "0.13", features = ["crypto", "pem", "x509-parser"] } webpki-roots = "1" rustls-native-certs = "0.8" diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 9f51301759..f76258ba04 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -164,6 +164,17 @@ pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; /// by workload child processes. pub const PROXY_TLS_DIR: &str = "OPENSHELL_PROXY_TLS_DIR"; +/// Optional path to a durable PEM-encoded interception CA certificate. +/// Must be configured together with [`PROXY_CA_KEY`]. +pub const PROXY_CA_CERT: &str = "OPENSHELL_PROXY_CA_CERT"; + +/// Optional path to the private key for [`PROXY_CA_CERT`]. +/// Must be configured together with the certificate path. +pub const PROXY_CA_KEY: &str = "OPENSHELL_PROXY_CA_KEY"; + +/// Whether the control-owned SSH Unix socket is shared across trusted UIDs. +pub const SSH_SOCKET_SHARED: &str = "OPENSHELL_SSH_SOCKET_SHARED"; + /// Path to the CA certificate for mTLS communication with the gateway. pub const TLS_CA: &str = "OPENSHELL_TLS_CA"; diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index c30c0b4273..0d7061786b 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -204,6 +204,15 @@ fn has_network_runtime_capability(capabilities: Option<&str>, required: &str) -> const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; +fn shared_ssh_socket_from_env() -> bool { + std::env::var(openshell_core::sandbox_env::SSH_SOCKET_SHARED) + .is_ok_and(|value| shared_ssh_socket_value(&value)) +} + +fn shared_ssh_socket_value(value: &str) -> bool { + value == "1" || value.eq_ignore_ascii_case("true") +} + /// Run a command in the sandbox. /// /// # Errors @@ -315,7 +324,7 @@ pub async fn run_sandbox( MiddlewareRegistryStatus::Synchronized, loaded_policy_origin, bootstrap.agent_proposals_enabled, - false, + shared_ssh_socket_from_env(), ) } else { load_policy( @@ -4845,6 +4854,15 @@ fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String mod tests { use super::*; + #[test] + fn shared_ssh_socket_setting_is_explicit() { + assert!(shared_ssh_socket_value("1")); + assert!(shared_ssh_socket_value("true")); + assert!(shared_ssh_socket_value("TRUE")); + assert!(!shared_ssh_socket_value("0")); + assert!(!shared_ssh_socket_value("yes")); + } + #[tokio::test] async fn control_readiness_exists_only_while_guard_is_live() { let readiness = ControlReadiness::start_at("127.0.0.1:0".parse().unwrap()) diff --git a/crates/openshell-supervisor-network/src/l7/tls.rs b/crates/openshell-supervisor-network/src/l7/tls.rs index d3def44743..7c8301f8a7 100644 --- a/crates/openshell-supervisor-network/src/l7/tls.rs +++ b/crates/openshell-supervisor-network/src/l7/tls.rs @@ -8,7 +8,7 @@ //! store, terminates TLS from the client (presenting dynamic certs per hostname), //! inspects the plaintext HTTP, then re-encrypts to upstream using real root CAs. -use miette::{IntoDiagnostic, Result, miette}; +use miette::{IntoDiagnostic, Result, WrapErr, miette}; use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName}; use rustls::{ClientConfig, ServerConfig}; @@ -66,6 +66,75 @@ impl SandboxCa { pub fn cert_pem(&self) -> &str { &self.ca_cert_pem } + + /// Returns the CA private key in PKCS#8 PEM format. + pub fn private_key_pem(&self) -> String { + self.ca_key.serialize_pem() + } + + /// Load a durable CA certificate and matching private key from absolute paths. + pub fn load_from_paths(certificate_path: &Path, private_key_path: &Path) -> Result { + if !certificate_path.is_absolute() || !private_key_path.is_absolute() { + return Err(miette!( + "proxy CA certificate and key paths must be absolute" + )); + } + if certificate_path == private_key_path { + return Err(miette!( + "proxy CA certificate and private key must use different paths" + )); + } + let certificate_pem = std::fs::read_to_string(certificate_path) + .into_diagnostic() + .wrap_err_with(|| { + format!("read proxy CA certificate {}", certificate_path.display()) + })?; + let private_key_pem = std::fs::read_to_string(private_key_path) + .into_diagnostic() + .wrap_err_with(|| { + format!("read proxy CA private key {}", private_key_path.display()) + })?; + Self::from_pem(&certificate_pem, &private_key_pem) + } + + /// Load a durable CA while preserving the exact certificate bytes supplied + /// by the provisioner for boundary launch replay. + pub fn from_pem(certificate_pem: &str, private_key_pem: &str) -> Result { + let ca_key = KeyPair::from_pem(private_key_pem) + .into_diagnostic() + .wrap_err("parse proxy CA private key")?; + let certificates = rustls_pemfile::certs(&mut certificate_pem.as_bytes()) + .collect::, _>>() + .into_diagnostic() + .wrap_err("parse proxy CA certificate")?; + if certificates.len() != 1 { + return Err(miette!( + "proxy CA certificate file must contain exactly one certificate" + )); + } + let private_key = rustls_pemfile::private_key(&mut private_key_pem.as_bytes()) + .into_diagnostic() + .wrap_err("parse proxy CA private key")? + .ok_or_else(|| miette!("proxy CA private key file contains no private key"))?; + ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, private_key) + .into_diagnostic() + .wrap_err("proxy CA certificate and private key do not match")?; + + let params = CertificateParams::from_ca_cert_pem(certificate_pem) + .into_diagnostic() + .wrap_err("parse proxy CA signing certificate")?; + let ca_cert = params + .self_signed(&ca_key) + .into_diagnostic() + .wrap_err("initialize proxy CA signer")?; + Ok(Self { + ca_cert, + ca_key, + ca_cert_pem: certificate_pem.to_string(), + }) + } } /// A leaf certificate chain and private key for a specific hostname. @@ -561,4 +630,23 @@ mod tests { "bundle should contain at least one cert", ); } + + #[test] + fn durable_ca_round_trip_preserves_certificate_bytes() { + let generated = SandboxCa::generate().unwrap(); + let certificate = generated.cert_pem().to_string(); + let private_key = generated.private_key_pem(); + let loaded = SandboxCa::from_pem(&certificate, &private_key).unwrap(); + + assert_eq!(loaded.cert_pem(), certificate); + assert_eq!(loaded.private_key_pem(), private_key); + } + + #[test] + fn durable_ca_rejects_mismatched_key_and_relative_paths() { + let certificate = SandboxCa::generate().unwrap(); + let other_key = SandboxCa::generate().unwrap(); + assert!(SandboxCa::from_pem(certificate.cert_pem(), &other_key.private_key_pem()).is_err()); + assert!(SandboxCa::load_from_paths(Path::new("ca.pem"), Path::new("ca.key")).is_err()); + } } diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index e761bb57e7..e85ba4021f 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -318,10 +318,29 @@ pub async fn run_networking( // the proxy, so it's owned here. let identity_cache = opa_engine.map(|_| Arc::new(BinaryIdentityCache::new())); - // Generate ephemeral CA and TLS state for HTTPS L7 inspection. + // Load a provisioned CA when the boundary lifetime outlives this control + // process; otherwise generate an ephemeral CA. // The CA cert is written to disk so sandbox processes can trust it. let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { - match SandboxCa::generate() { + let configured_ca = match ( + std::env::var_os(openshell_core::sandbox_env::PROXY_CA_CERT), + std::env::var_os(openshell_core::sandbox_env::PROXY_CA_KEY), + ) { + (Some(certificate), Some(private_key)) => Some(SandboxCa::load_from_paths( + std::path::Path::new(&certificate), + std::path::Path::new(&private_key), + )?), + (None, None) => None, + _ => { + return Err(miette::miette!( + "{} and {} must be configured together", + openshell_core::sandbox_env::PROXY_CA_CERT, + openshell_core::sandbox_env::PROXY_CA_KEY, + )); + } + }; + let durable_ca = configured_ca.is_some(); + match configured_ca.map_or_else(SandboxCa::generate, Ok) { Ok(ca) => { let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) .unwrap_or_else(|_| openshell_core::container_paths::TLS_ROOT.to_string()); @@ -361,7 +380,11 @@ pub async fn run_networking( .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Enabled, "enabled") - .message("TLS termination enabled: ephemeral CA generated") + .message(if durable_ca { + "TLS termination enabled: provisioned CA loaded" + } else { + "TLS termination enabled: ephemeral CA generated" + }) .build() ); (Some(state), Some(paths)) From e55ff6076bd988475079a24b3fc03b0d026856ec Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 03:17:47 -0700 Subject: [PATCH 23/37] fix(supervisor): honor shared boundary socket setting Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 0d7061786b..c37fe56c1c 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1073,7 +1073,7 @@ pub async fn run_sandbox( sandbox_id.as_deref(), openshell_endpoint.as_deref(), ssh_socket_path.as_deref(), - false, + shared_ssh_socket_from_env(), networking .as_ref() .and_then(|runtime| runtime.ca_file_paths.clone()), From 3aa93e130d41c243e408841fd6cf24bc1f98860f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 04:14:15 -0700 Subject: [PATCH 24/37] fix(supervisor): reconcile opaque provider revisions Signed-off-by: Drew Newberry --- .../src/provider_credentials.rs | 30 ++++-- .../src/boundary_protocol.rs | 36 ++++++-- .../src/remote.rs | 61 +++++++----- .../openshell-sandbox/src/boundary_server.rs | 92 +++++++++++++++++-- 4 files changed, 167 insertions(+), 52 deletions(-) diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index 784f263ed2..d9055fa319 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -428,11 +428,14 @@ impl ProviderCredentialState { (inner.current.revision, env) } - /// Install a newer workload-facing environment snapshot and return the - /// effective revision. Stale and duplicate updates are idempotently - /// ignored under the same write lock that installs the snapshot. - pub fn install_child_env_snapshot_if_newer( + /// Compare and install a workload-facing environment snapshot. + /// + /// Provider environment revisions are opaque content identities, not + /// ordered counters. The expected revision makes retries idempotent while + /// rejecting updates based on a stale view of the boundary state. + pub fn compare_and_install_child_env_snapshot( &self, + expected_revision: u64, revision: u64, mut child_env: HashMap, ) -> u64 { @@ -440,7 +443,7 @@ impl ProviderCredentialState { .inner .write() .expect("provider credential state poisoned"); - if revision <= inner.current.revision { + if revision == inner.current.revision || expected_revision != inner.current.revision { return inner.current.revision; } @@ -2163,30 +2166,37 @@ mod tests { } #[test] - fn child_env_snapshot_update_is_monotonic_and_atomic() { + fn child_env_snapshot_update_uses_opaque_revision_cas() { let state = ProviderCredentialState::from_child_env_snapshot( 4, HashMap::from([("TOKEN".to_string(), "four".to_string())]), ); assert_eq!( - state.install_child_env_snapshot_if_newer( + state.compare_and_install_child_env_snapshot( + 4, 6, HashMap::from([("TOKEN".to_string(), "six".to_string())]), ), 6 ); assert_eq!( - state.install_child_env_snapshot_if_newer( + state.compare_and_install_child_env_snapshot( + 4, 5, HashMap::from([("TOKEN".to_string(), "stale".to_string())]), ), 6 ); + assert_eq!( + state.compare_and_install_child_env_snapshot(6, 2, HashMap::new()), + 2, + "opaque revisions may move numerically backwards" + ); let (revision, env) = state.child_env_snapshot_with_gcp_resolved(); - assert_eq!(revision, 6); - assert_eq!(env.get("TOKEN").map(String::as_str), Some("six")); + assert_eq!(revision, 2); + assert!(env.is_empty(), "an empty snapshot must revoke the old env"); } #[test] diff --git a/crates/openshell-isolation-interface/src/boundary_protocol.rs b/crates/openshell-isolation-interface/src/boundary_protocol.rs index c439f402c4..231106be80 100644 --- a/crates/openshell-isolation-interface/src/boundary_protocol.rs +++ b/crates/openshell-isolation-interface/src/boundary_protocol.rs @@ -35,7 +35,7 @@ pub const STREAM_EXIT: u8 = 3; pub const STREAM_STDIN_CLOSED: u8 = 4; pub const MAX_STREAM_FRAME_BYTES: usize = 64 * 1024; /// Version of the driver-to-supervisor boundary descriptor and config shape. -pub const BOUNDARY_PROTOCOL_VERSION: u32 = 2; +pub const BOUNDARY_PROTOCOL_VERSION: u32 = 3; /// Control-side endpoint for a driver-provisioned boundary. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -303,6 +303,7 @@ pub enum Request { provider_env: std::collections::HashMap, }, UpdateProviderEnvironment { + expected_revision: u64, revision: u64, provider_env: std::collections::HashMap, }, @@ -372,10 +373,12 @@ impl fmt::Debug for Request { ) .finish(), Self::UpdateProviderEnvironment { + expected_revision, revision, provider_env, } => formatter .debug_struct("UpdateProviderEnvironment") + .field("expected_revision", expected_revision) .field("revision", revision) .field( "provider_env_keys", @@ -436,17 +439,34 @@ pub struct ResponseEnvelope { pub enum Response { Attached, Confirmed, - Started { process_id: String }, - ProviderEnvironmentUpdated { revision: u64 }, - ProcessAttached { terminal: bool }, - Exited { status: ExitStatusWire }, + Started { + process_id: String, + provider_env_revision: u64, + }, + ProviderEnvironmentUpdated { + revision: u64, + }, + ProcessAttached { + terminal: bool, + }, + Exited { + status: ExitStatusWire, + }, Signaled, Terminated, - ExecStarted { process_id: String, pty: bool }, + ExecStarted { + process_id: String, + pty: bool, + }, Resized, PortConnected, - NetworkConnected { identity: BinaryIdentityWire }, - Error { kind: String, message: String }, + NetworkConnected { + identity: BinaryIdentityWire, + }, + Error { + kind: String, + message: String, + }, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index 58465a918f..f2fc128ceb 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -301,7 +301,11 @@ impl ReadyBoundary for RemoteReady { provider_env, }) .await?; - let Response::Started { process_id } = response else { + let Response::Started { + process_id, + provider_env_revision, + } = response + else { return Err(unexpected_response("started", &response)); }; let process = Arc::new(RemoteProcess { @@ -313,6 +317,7 @@ impl ReadyBoundary for RemoteReady { exec: Arc::new(RemoteExec { client: self.client.clone(), provider_credentials: self.provider_credentials, + boundary_revision: tokio::sync::Mutex::new(provider_env_revision), }), port_forward: Arc::new(RemotePortForward { client: self.client, @@ -467,36 +472,42 @@ async fn pump_process_responses( struct RemoteExec { client: Arc, provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, + boundary_revision: tokio::sync::Mutex, } #[async_trait] impl BoundaryExec for RemoteExec { async fn exec(&self, spec: ExecSpec) -> Result { - let (revision, provider_env) = self - .provider_credentials - .child_env_snapshot_with_gcp_resolved(); - let response = self - .client - .call_idempotent(Request::UpdateProviderEnvironment { - revision, - provider_env, - }) - .await?; - let Response::ProviderEnvironmentUpdated { - revision: effective_revision, - } = response - else { - return Err(unexpected_response( - "provider_environment_updated", - &response, - )); - }; - if effective_revision < revision { - return Err(BackendError::Process(format!( - "boundary provider environment revision {effective_revision} is older than requested revision {revision}" - ))); + let mut boundary_revision = self.boundary_revision.lock().await; + for _ in 0..3 { + let (revision, provider_env) = self + .provider_credentials + .child_env_snapshot_with_gcp_resolved(); + let response = self + .client + .call_idempotent(Request::UpdateProviderEnvironment { + expected_revision: *boundary_revision, + revision, + provider_env, + }) + .await?; + let Response::ProviderEnvironmentUpdated { + revision: effective_revision, + } = response + else { + return Err(unexpected_response( + "provider_environment_updated", + &response, + )); + }; + *boundary_revision = effective_revision; + if effective_revision == revision { + return open_exec_session(self.client.clone(), spec).await; + } } - open_exec_session(self.client.clone(), spec).await + Err(BackendError::Process( + "boundary provider environment changed concurrently during reconciliation".to_string(), + )) } } diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index a4ecb84c13..22520027cb 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -478,6 +478,19 @@ mod linux { provider_env: std::collections::HashMap, } + impl StartedAgent { + /// Provider environment is mutable runtime state. A replacement + /// control must replay every immutable launch input exactly, then + /// reconcile the current provider snapshot through the CAS update. + fn matches_replay(&self, other: &Self) -> bool { + self.sandbox_id == other.sandbox_id + && self.spec == other.spec + && self.policy == other.policy + && self.ca_cert == other.ca_cert + && self.ca_bundle == other.ca_bundle + } + } + struct MainAttachment { process: Arc, session: Arc, @@ -669,9 +682,10 @@ mod linux { provider_env, ), Request::UpdateProviderEnvironment { + expected_revision, revision, provider_env, - } => self.update_provider_environment(revision, provider_env), + } => self.update_provider_environment(expected_revision, revision, provider_env), Request::Wait { process_id } => self.wait(&process_id), Request::Signal { process_id, signal } => self.signal(&process_id, signal), Request::Terminate { process_id } => self.terminate(&process_id), @@ -935,9 +949,13 @@ mod linux { provider_env: provider_env.clone(), }; if let RuntimeState::Running(process) = &*state { - return if lock(&self.started_agent).as_ref() == Some(&requested) { + return if lock(&self.started_agent) + .as_ref() + .is_some_and(|accepted| accepted.matches_replay(&requested)) + { Response::Started { process_id: process.process_id(), + provider_env_revision: process.provider_credentials.snapshot().revision, } } else { guest_error( @@ -985,11 +1003,15 @@ mod linux { let process_id = process.process_id(); *lock(&self.started_agent) = Some(requested); *state = RuntimeState::Running(process); - Response::Started { process_id } + Response::Started { + process_id, + provider_env_revision, + } } fn update_provider_environment( &self, + expected_revision: u64, revision: u64, provider_env: std::collections::HashMap, ) -> Response { @@ -1005,7 +1027,7 @@ mod linux { }; let revision = process .provider_credentials - .install_child_env_snapshot_if_newer(revision, provider_env); + .compare_and_install_child_env_snapshot(expected_revision, revision, provider_env); Response::ProviderEnvironmentUpdated { revision } } @@ -2063,7 +2085,11 @@ mod linux { std::collections::HashMap::new(), ) }; - let Response::Started { process_id } = start() else { + let Response::Started { + process_id, + provider_env_revision: 0, + } = start() + else { panic!("initial start did not succeed"); }; @@ -2093,7 +2119,8 @@ mod linux { assert_eq!( start(), Response::Started { - process_id: process_id.clone() + process_id: process_id.clone(), + provider_env_revision: 0, } ); @@ -2245,20 +2272,22 @@ mod linux { assert_eq!( boundary.start_agent( "sandbox-retained".to_string(), - agent_spec, - wire_policy, + agent_spec.clone(), + wire_policy.clone(), None, None, 0, std::collections::HashMap::new(), ), Response::Started { - process_id: process.process_id() + process_id: process.process_id(), + provider_env_revision: 0, } ); assert_eq!( boundary.update_provider_environment( + 0, 2, std::collections::HashMap::from([( "ROTATED_TOKEN".to_string(), @@ -2269,6 +2298,7 @@ mod linux { ); assert_eq!( boundary.update_provider_environment( + 0, 1, std::collections::HashMap::from([( "ROTATED_TOKEN".to_string(), @@ -2277,6 +2307,50 @@ mod linux { ), Response::ProviderEnvironmentUpdated { revision: 2 } ); + assert_eq!( + boundary.update_provider_environment(2, 1, std::collections::HashMap::new()), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a numerically smaller opaque revision must revoke the environment" + ); + assert_eq!( + boundary.update_provider_environment( + 2, + 3, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "out-of-order".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a stale expected revision must not overwrite current state" + ); + assert_eq!( + boundary.update_provider_environment(1, 1, std::collections::HashMap::new()), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a duplicate update must be idempotent" + ); + + assert_eq!(boundary.attach(wire_policy.clone()), Response::Attached); + assert_eq!(boundary.confirm(), Response::Confirmed); + assert_eq!( + boundary.start_agent( + "sandbox-retained".to_string(), + agent_spec, + wire_policy, + None, + None, + 99, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "replacement-control-snapshot".to_string(), + )]), + ), + Response::Started { + process_id: process.process_id(), + provider_env_revision: 1, + }, + "a replacement control must resume from the boundary's current revision" + ); let (cancelled_id, cancelled_session) = boundary .start_exec(ExecSpecWire { From cc2ff951def21c27e3be2f18920e1867a75efc3c Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 04:39:47 -0700 Subject: [PATCH 25/37] fix(supervisor): expose public proxy CA material Signed-off-by: Drew Newberry --- .../openshell-sandbox/src/boundary_server.rs | 161 +++++++++++++++++- 1 file changed, 154 insertions(+), 7 deletions(-) diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 22520027cb..61937ce4bd 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -1141,15 +1141,107 @@ mod linux { ); } }; - let directory = std::path::PathBuf::from("/run/openshell/proxy-ca"); - std::fs::create_dir_all(&directory) - .map_err(|error| format!("create boundary proxy CA directory: {error}"))?; + install_ca_material_at(Path::new("/run/openshell/proxy-ca"), &ca_cert, &ca_bundle) + } + + fn install_ca_material_at( + directory: &Path, + ca_cert: &[u8], + ca_bundle: &[u8], + ) -> Result, String> { + use std::io::Write as _; + use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _}; + + let parent = directory + .parent() + .ok_or_else(|| "boundary proxy CA directory has no parent".to_string())?; + for (path, public_directory) in [(parent, false), (directory, true)] { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "boundary proxy CA directory component is a symlink: {}", + path.display() + )); + } + Ok(metadata) if !metadata.is_dir() => { + return Err(format!( + "boundary proxy CA directory component is not a directory: {}", + path.display() + )); + } + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => { + std::fs::create_dir(path).map_err(|error| { + format!( + "create boundary proxy CA directory {}: {error}", + path.display() + ) + })?; + } + Err(error) => { + return Err(format!( + "inspect boundary proxy CA directory {}: {error}", + path.display() + )); + } + } + let current_mode = std::fs::metadata(path) + .map_err(|error| { + format!( + "inspect boundary proxy CA directory permissions {}: {error}", + path.display() + ) + })? + .permissions() + .mode(); + let mode = if public_directory { + 0o755 + } else { + current_mode | 0o111 + }; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).map_err( + |error| { + format!( + "set boundary proxy CA directory permissions {}: {error}", + path.display() + ) + }, + )?; + } let ca_path = directory.join("ca.crt"); let bundle_path = directory.join("ca-bundle.crt"); - std::fs::write(&ca_path, ca_cert) - .map_err(|error| format!("write boundary proxy CA: {error}"))?; - std::fs::write(&bundle_path, ca_bundle) - .map_err(|error| format!("write boundary proxy CA bundle: {error}"))?; + for (path, contents, label) in [ + (&ca_path, ca_cert, "boundary proxy CA"), + (&bundle_path, ca_bundle, "boundary proxy CA bundle"), + ] { + let temporary = path.with_extension("tmp"); + if let Ok(metadata) = std::fs::symlink_metadata(&temporary) { + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(format!( + "refusing unsafe temporary {label} path: {}", + temporary.display() + )); + } + std::fs::remove_file(&temporary) + .map_err(|error| format!("remove stale temporary {label}: {error}"))?; + } + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o444) + .custom_flags(libc::O_NOFOLLOW) + .open(&temporary) + .map_err(|error| format!("create temporary {label}: {error}"))?; + if let Err(error) = file + .write_all(contents) + .and_then(|()| file.sync_all()) + .and_then(|()| file.set_permissions(std::fs::Permissions::from_mode(0o444))) + .and_then(|()| std::fs::rename(&temporary, path)) + { + let _ = std::fs::remove_file(&temporary); + return Err(format!("install {label}: {error}")); + } + } Ok(Some((ca_path, bundle_path))) } @@ -1850,6 +1942,61 @@ mod linux { assert!(!debug.contains("never-log-this")); } + #[test] + fn installed_proxy_ca_is_readable_by_a_non_root_workload_identity() { + use std::os::unix::fs::PermissionsExt as _; + + let root = tempfile::tempdir().expect("temporary CA root"); + let directory = root.path().join("openshell/proxy-ca"); + let (ca_path, bundle_path) = install_ca_material_at( + &directory, + b"public test certificate", + b"public test bundle", + ) + .expect("install proxy CA") + .expect("CA paths"); + + assert_eq!( + std::fs::metadata(directory.parent().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o111, + 0o111, + "non-root workload identities must be able to traverse the private parent" + ); + assert_eq!( + std::fs::metadata(&directory).unwrap().permissions().mode() & 0o777, + 0o755, + "non-root workload identities must be able to traverse the CA directory" + ); + for (path, expected) in [ + (&ca_path, b"public test certificate".as_slice()), + (&bundle_path, b"public test bundle".as_slice()), + ] { + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o444, + "public CA material must be readable by the workload" + ); + assert_eq!(std::fs::read(path).unwrap(), expected); + } + } + + #[test] + fn proxy_ca_install_rejects_a_symlinked_directory() { + let root = tempfile::tempdir().expect("temporary CA root"); + let target = root.path().join("target"); + std::fs::create_dir(&target).unwrap(); + let parent = root.path().join("openshell"); + std::fs::create_dir(&parent).unwrap(); + std::os::unix::fs::symlink(&target, parent.join("proxy-ca")).unwrap(); + + let error = install_ca_material_at(&parent.join("proxy-ca"), b"certificate", b"bundle") + .expect_err("symlinked CA directory must fail closed"); + assert!(error.contains("symlink"), "unexpected error: {error}"); + } + #[test] fn constant_time_comparison_checks_length_and_content() { assert!(constant_time_eq(b"same", b"same")); From 9f01390d55cf1b9632e26ab3762adf1fad83725b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 04:53:38 -0700 Subject: [PATCH 26/37] fix(supervisor): use traversable proxy CA path Signed-off-by: Drew Newberry --- .../openshell-sandbox/src/boundary_server.rs | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 61937ce4bd..bc7a344df7 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -1141,7 +1141,7 @@ mod linux { ); } }; - install_ca_material_at(Path::new("/run/openshell/proxy-ca"), &ca_cert, &ca_bundle) + install_ca_material_at(Path::new("/run/openshell-proxy-ca"), &ca_cert, &ca_bundle) } fn install_ca_material_at( @@ -1155,7 +1155,7 @@ mod linux { let parent = directory .parent() .ok_or_else(|| "boundary proxy CA directory has no parent".to_string())?; - for (path, public_directory) in [(parent, false), (directory, true)] { + for path in [parent, directory] { match std::fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_symlink() => { return Err(format!( @@ -1194,19 +1194,21 @@ mod linux { })? .permissions() .mode(); - let mode = if public_directory { - 0o755 - } else { - current_mode | 0o111 - }; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).map_err( - |error| { - format!( - "set boundary proxy CA directory permissions {}: {error}", - path.display() - ) - }, - )?; + if path == directory { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).map_err( + |error| { + format!( + "set boundary proxy CA directory permissions {}: {error}", + path.display() + ) + }, + )?; + } else if current_mode & 0o111 != 0o111 { + return Err(format!( + "boundary proxy CA parent is not traversable by workload identities: {}", + path.display() + )); + } } let ca_path = directory.join("ca.crt"); let bundle_path = directory.join("ca-bundle.crt"); @@ -1947,7 +1949,8 @@ mod linux { use std::os::unix::fs::PermissionsExt as _; let root = tempfile::tempdir().expect("temporary CA root"); - let directory = root.path().join("openshell/proxy-ca"); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let directory = root.path().join("openshell-proxy-ca"); let (ca_path, bundle_path) = install_ca_material_at( &directory, b"public test certificate", @@ -1963,7 +1966,7 @@ mod linux { .mode() & 0o111, 0o111, - "non-root workload identities must be able to traverse the private parent" + "non-root workload identities must be able to traverse the full path" ); assert_eq!( std::fs::metadata(&directory).unwrap().permissions().mode() & 0o777, @@ -1988,12 +1991,15 @@ mod linux { let root = tempfile::tempdir().expect("temporary CA root"); let target = root.path().join("target"); std::fs::create_dir(&target).unwrap(); - let parent = root.path().join("openshell"); - std::fs::create_dir(&parent).unwrap(); - std::os::unix::fs::symlink(&target, parent.join("proxy-ca")).unwrap(); + let parent = root.path(); + std::os::unix::fs::symlink(&target, parent.join("openshell-proxy-ca")).unwrap(); - let error = install_ca_material_at(&parent.join("proxy-ca"), b"certificate", b"bundle") - .expect_err("symlinked CA directory must fail closed"); + let error = install_ca_material_at( + &parent.join("openshell-proxy-ca"), + b"certificate", + b"bundle", + ) + .expect_err("symlinked CA directory must fail closed"); assert!(error.contains("symlink"), "unexpected error: {error}"); } From 833294e8aed4d2cb3ab0e0f4384af05f81f70b9e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 04:53:38 -0700 Subject: [PATCH 27/37] fix(gateway): retain terminal sandbox relay access Signed-off-by: Drew Newberry --- crates/openshell-server/src/grpc/sandbox.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index d4d08da251..0f3a2402d6 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -2196,10 +2196,7 @@ fn sandbox_relay_reachable(state: &ServerState, sandbox: &Sandbox) -> bool { let phase = SandboxPhase::try_from(sandbox.phase()).ok(); matches!(phase, Some(SandboxPhase::Ready)) || (matches!(phase, Some(SandboxPhase::Completed | SandboxPhase::Error)) - && state.supervisor_sessions.has_session(sandbox.object_id()) - && !state - .supervisor_sessions - .terminal_delivery_finalized(sandbox.object_id())) + && state.supervisor_sessions.has_session(sandbox.object_id())) } pub(super) async fn handle_create_ssh_session( @@ -5465,6 +5462,9 @@ mod tests { .supervisor_sessions .finalize_main_process_exit("sandbox-work") ); + assert!(sandbox_relay_reachable(&state, &sandbox)); + + assert!(state.supervisor_sessions.disconnect("sandbox-work")); assert!(!sandbox_relay_reachable(&state, &sandbox)); } From 9b116725249e822750a31ec40dccce0d88c71049 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 05:06:49 -0700 Subject: [PATCH 28/37] fix(supervisor): admit runtime CA through Landlock Signed-off-by: Drew Newberry --- crates/openshell-server/src/grpc/sandbox.rs | 9 ++ .../src/boundary_exec.rs | 11 +- .../src/process.rs | 117 +++++++++++++++++- .../openshell-supervisor-process/src/ssh.rs | 28 ++++- 4 files changed, 153 insertions(+), 12 deletions(-) diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 0f3a2402d6..760e54fc0e 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -6228,7 +6228,16 @@ mod tests { let mut sandbox = test_sandbox("cross-ws", Vec::new()); sandbox.metadata.as_mut().unwrap().workspace = "other-workspace".to_string(); + sandbox.set_phase(SandboxPhase::Completed as i32); state.store.put_message(&sandbox).await.unwrap(); + let (tx, _rx) = mpsc::channel(1); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + let _ = state.supervisor_sessions.register( + sandbox.object_id().to_string(), + "retained-terminal-session".to_string(), + tx, + shutdown_tx, + ); // --- handle_watch_sandbox --- let err = handle_watch_sandbox( diff --git a/crates/openshell-supervisor-process/src/boundary_exec.rs b/crates/openshell-supervisor-process/src/boundary_exec.rs index 8b3f81f92a..0ce2e5aa48 100644 --- a/crates/openshell-supervisor-process/src/boundary_exec.rs +++ b/crates/openshell-supervisor-process/src/boundary_exec.rs @@ -106,8 +106,15 @@ impl LocalBoundaryExec { if self.enforcement_mode.enforces_child_sandbox() { crate::sandbox::linux::log_sandbox_readiness(&self.policy, workdir); } - crate::process::prepare_child_sandbox(&self.policy, workdir, self.enforcement_mode) - .map_err(|error| BackendError::Process(error.to_string())) + let runtime_read_only = + crate::process::ca_runtime_read_only_paths(self.ca_file_paths.as_deref()); + crate::process::prepare_child_sandbox( + &self.policy, + workdir, + self.enforcement_mode, + &runtime_read_only, + ) + .map_err(|error| BackendError::Process(error.to_string())) } fn spawn_piped(&self, spec: &ExecSpec) -> Result { diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 3da9c6ad18..2829f7bbf1 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -147,19 +147,48 @@ pub(crate) fn prepare_child_sandbox( policy: &SandboxPolicy, workdir: Option<&str>, enforcement_mode: ProcessEnforcementMode, + runtime_read_only: &[PathBuf], ) -> Result> { if !enforcement_mode.enforces_child_sandbox() { return Ok(None); } + let effective_policy = policy_with_runtime_read_only(policy, runtime_read_only); + let prepared = if enforcement_mode.uses_privileged_process_setup() { - sandbox::linux::prepare(policy, workdir) + sandbox::linux::prepare(&effective_policy, workdir) } else { - sandbox::linux::prepare_current_user(policy, workdir) + sandbox::linux::prepare_current_user(&effective_policy, workdir) }?; Ok(Some(prepared)) } +fn policy_with_runtime_read_only( + policy: &SandboxPolicy, + runtime_read_only: &[PathBuf], +) -> SandboxPolicy { + let mut effective_policy = policy.clone(); + for path in runtime_read_only { + if !effective_policy.filesystem.read_only.contains(path) { + effective_policy.filesystem.read_only.push(path.clone()); + } + } + effective_policy +} + +pub(crate) fn ca_runtime_read_only_paths(ca_paths: Option<&(PathBuf, PathBuf)>) -> Vec { + let Some((certificate, bundle)) = ca_paths else { + return Vec::new(); + }; + let mut paths = Vec::with_capacity(3); + if let Some(directory) = certificate.parent() { + paths.push(directory.to_path_buf()); + } + paths.push(certificate.clone()); + paths.push(bundle.clone()); + paths +} + const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::OCI_IMAGE_USER, openshell_core::sandbox_env::SANDBOX_UID, @@ -844,8 +873,14 @@ impl ProcessHandle { // runs as the sandbox UID, so inaccessible paths are unavailable to // the workload and best-effort compatibility skips them. #[cfg(target_os = "linux")] - let prepared_sandbox = prepare_child_sandbox(policy, workspace.root(), enforcement_mode) - .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; + let runtime_read_only = ca_runtime_read_only_paths(ca_paths); + let prepared_sandbox = prepare_child_sandbox( + policy, + workspace.root(), + enforcement_mode, + &runtime_read_only, + ) + .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; // Set up process group for signal handling (non-interactive mode only). // In interactive mode, we inherit the parent's process group to maintain // proper terminal control for shells and interactive programs. @@ -3456,6 +3491,80 @@ mod tests { } } + #[test] + fn runtime_ca_paths_are_added_to_the_effective_read_only_policy() { + let mut policy = policy_with_process(ProcessPolicy::default()); + policy.filesystem.read_only = vec![PathBuf::from("/usr")]; + let certificate = PathBuf::from("/run/openshell-proxy-ca/ca.crt"); + let bundle = PathBuf::from("/run/openshell-proxy-ca/ca-bundle.crt"); + + let effective = policy_with_runtime_read_only( + &policy, + &[certificate.clone(), bundle.clone(), certificate.clone()], + ); + + assert_eq!(policy.filesystem.read_only, vec![PathBuf::from("/usr")]); + assert_eq!( + effective.filesystem.read_only, + vec![PathBuf::from("/usr"), certificate, bundle] + ); + } + + #[cfg(target_os = "linux")] + #[test] + #[allow(unsafe_code)] + fn runtime_ca_material_remains_readable_after_landlock_for_non_root_workload() { + let root = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let ca_directory = root.path().join("openshell-proxy-ca"); + std::fs::create_dir(&ca_directory).unwrap(); + std::fs::set_permissions(&ca_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let certificate = ca_directory.join("ca.crt"); + let bundle = ca_directory.join("ca-bundle.crt"); + let denied = root.path().join("not-authorized"); + for path in [&certificate, &bundle, &denied] { + std::fs::write(path, b"public certificate material").unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o444)).unwrap(); + } + + let mut policy = policy_with_process(ProcessPolicy::default()); + policy.landlock = LandlockPolicy { + compatibility: openshell_core::policy::LandlockCompatibility::HardRequirement, + }; + let runtime_paths = + ca_runtime_read_only_paths(Some(&(certificate.clone(), bundle.clone()))); + let Ok(Some(prepared)) = + prepare_child_sandbox(&policy, None, ProcessEnforcementMode::Full, &runtime_paths) + else { + return; + }; + + match unsafe { fork() }.expect("fork should succeed") { + ForkResult::Child => { + let dropped = if nix::unistd::geteuid().is_root() { + unsafe { + libc::setgroups(0, std::ptr::null()) == 0 + && libc::setgid(42_235) == 0 + && libc::setuid(42_234) == 0 + } + } else { + true + }; + let valid = dropped + && sandbox::linux::enforce(prepared).is_ok() + && std::fs::read(&certificate).is_ok() + && std::fs::read(&bundle).is_ok() + && std::fs::read(&denied).is_err(); + unsafe { libc::_exit(i32::from(!valid)) }; + } + ForkResult::Parent { child } => assert_eq!( + waitpid(child, None).expect("waitpid should succeed"), + WaitStatus::Exited(child, 0), + "Landlock must preserve non-root access only to admitted public CA material" + ), + } + } + #[cfg(unix)] #[test] fn validate_oci_workspace_rejects_restrictive_parent() { diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index e82e176942..d06903ad0b 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -1340,9 +1340,17 @@ fn spawn_pty_shell( // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] - let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workdir.as_deref(), enforcement_mode) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; + let prepared_sandbox = { + let runtime_read_only = + crate::process::ca_runtime_read_only_paths(ca_file_paths.as_deref()); + crate::process::prepare_child_sandbox( + policy, + workdir.as_deref(), + enforcement_mode, + &runtime_read_only, + ) + .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))? + }; #[cfg(unix)] { @@ -1506,9 +1514,17 @@ fn spawn_pipe_exec( // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] - let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workdir.as_deref(), enforcement_mode) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; + let prepared_sandbox = { + let runtime_read_only = + crate::process::ca_runtime_read_only_paths(ca_file_paths.as_deref()); + crate::process::prepare_child_sandbox( + policy, + workdir.as_deref(), + enforcement_mode, + &runtime_read_only, + ) + .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))? + }; #[cfg(unix)] { From b6fd4da5adbd554768fb60f81848cf817c3b6acb Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 05:14:25 -0700 Subject: [PATCH 29/37] fix(supervisor): publish readiness after relay registration Signed-off-by: Drew Newberry --- .../src/delegated.rs | 37 ++++++++++++++----- .../src/supervisor_session.rs | 31 +++++++++++++++- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index db0ea9b56e..60dbf7848f 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -161,15 +161,34 @@ pub async fn start_boundary_access( } let session_task = match (openshell_endpoint, sandbox_id) { - (Some(endpoint), Some(id)) => Some(crate::supervisor_session::spawn( - endpoint.to_string(), - id.to_string(), - ssh_socket_path, - port_forward, - None, - terminating.clone(), - instance_id.clone(), - )), + (Some(endpoint), Some(id)) => { + let (task, mut accepted) = crate::supervisor_session::spawn_with_readiness( + endpoint.to_string(), + id.to_string(), + ssh_socket_path, + port_forward, + None, + terminating.clone(), + instance_id.clone(), + ); + match tokio::time::timeout(Duration::from_secs(10), accepted.wait_for(|ready| *ready)) + .await + { + Ok(Ok(_)) => Some(task), + Ok(Err(_)) => { + task.abort(); + return Err(miette::miette!( + "supervisor session ended before gateway acceptance" + )); + } + Err(_) => { + task.abort(); + return Err(miette::miette!( + "gateway did not accept supervisor session within 10 seconds" + )); + } + } + } _ => None, }; diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index db01d07323..203c06d17b 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -281,6 +281,32 @@ pub fn spawn( terminating: Arc, instance_id: String, ) -> tokio::task::JoinHandle<()> { + spawn_with_readiness( + endpoint, + sandbox_id, + ssh_socket_path, + port_forward, + expected_ssh_peer_pid, + terminating, + instance_id, + ) + .0 +} + +/// Spawn the supervisor session and expose when the gateway has accepted it. +pub fn spawn_with_readiness( + endpoint: String, + sandbox_id: String, + ssh_socket_path: std::path::PathBuf, + port_forward: Arc, + expected_ssh_peer_pid: Option, + terminating: Arc, + instance_id: String, +) -> ( + tokio::task::JoinHandle<()>, + tokio::sync::watch::Receiver, +) { + let (ready_tx, ready_rx) = tokio::sync::watch::channel(false); let config = SessionConfig { endpoint, sandbox_id, @@ -289,8 +315,9 @@ pub fn spawn( expected_ssh_peer_pid, terminating, instance_id, + ready_tx, }; - tokio::spawn(run_session_loop(config)) + (tokio::spawn(run_session_loop(config)), ready_rx) } struct SessionConfig { @@ -301,6 +328,7 @@ struct SessionConfig { expected_ssh_peer_pid: Option, terminating: Arc, instance_id: String, + ready_tx: tokio::sync::watch::Sender, } async fn run_session_loop(config: SessionConfig) { @@ -390,6 +418,7 @@ async fn run_single_session( heartbeat_secs, ); ocsf_emit!(event); + config.ready_tx.send_replace(true); // Main loop: receive gateway messages + send heartbeats. let mut heartbeat_interval = From 9092db0953620695bf05f09b82641a2077adc806 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 05:19:20 -0700 Subject: [PATCH 30/37] fix(supervisor): retain expected remote main attachment Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/lib.rs | 4 ++- .../src/delegated.rs | 30 +++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index c37fe56c1c..11d8e2fe83 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1138,7 +1138,9 @@ pub async fn run_sandbox( if !retain_access { control_readiness.take(); } - boundary_access.publish_main_exit(exit_code).await; + boundary_access + .publish_main_exit(exit_code, await_main_process_attachment) + .await; // `shutdown_requested` has already completed when shutdown won the // lifecycle select above and must not be polled again. let mut completion_cancelled = !retain_access; diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index 60dbf7848f..55bdf01e06 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -50,11 +50,13 @@ impl BoundaryAccess { } /// Publish the canonical process's terminal status to attached clients. - pub async fn publish_main_exit(&self, exit_code: i32) { + pub async fn publish_main_exit(&self, exit_code: i32, attachment_expected: bool) { let Some(main_session) = self.main_session.as_ref() else { return; }; - let _ = main_session.finish_remote(exit_code, false).await; + let _ = main_session + .finish_remote(exit_code, attachment_expected) + .await; } /// Release terminal delivery after the gateway acknowledges the exit, then @@ -566,3 +568,27 @@ pub async fn finalize_main_process_exit(endpoint: &str, sandbox_id: &str, instan } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn expected_post_exit_attachment_is_preserved_for_remote_main() { + let main_session = crate::main_session::MainSession::inert(); + let access = BoundaryAccess { + instance_id: "instance".to_string(), + terminating: Arc::new(AtomicBool::new(false)), + ssh_task: None, + session_task: None, + main_session: Some(main_session.clone()), + }; + + access.publish_main_exit(7, true).await; + + main_session + .begin_terminal_attachment() + .expect("declared CLI attachment must remain valid after a fast remote main exits"); + main_session.end_terminal_attachment(); + } +} From b6848621723e39110f40b08128d4710b42b7e7fc Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 05:53:33 -0700 Subject: [PATCH 31/37] test(sandbox): assert reconciled provider revocation Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/boundary_server.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index bc7a344df7..e1f8bce0aa 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -2542,7 +2542,8 @@ mod linux { program: "/bin/sh".to_string(), args: vec![ "-c".to_string(), - "printf %s \"$ROTATED_TOKEN\"".to_string(), + "if [ -z \"${ROTATED_TOKEN+x}\" ]; then printf revoked; else printf 'unexpected:%s' \"$ROTATED_TOKEN\"; fi" + .to_string(), ], env: Vec::new(), workdir: None, @@ -2556,7 +2557,10 @@ mod linux { process_runtime .block_on(session.stdout.read_to_string(&mut output)) .expect("read retained exec output"); - assert_eq!(output, "refreshed"); + assert_eq!( + output, "revoked", + "exec after canonical exit must use the latest reconciled provider snapshot" + ); assert!(matches!( process_runtime.block_on(session.process.wait()), Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(0)) From dbe4f37dcf3ec24e083e2563305e0e27da81aa4f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 05:55:28 -0700 Subject: [PATCH 32/37] chore(isolation): move VM packaging to driver layer Signed-off-by: Drew Newberry --- e2e/rust/e2e-vm.sh | 8 ++-- tasks/scripts/vm/build-supervisor-bundle.sh | 46 --------------------- 2 files changed, 4 insertions(+), 50 deletions(-) diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 3805a2d092..96da2a879f 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -88,10 +88,10 @@ if [ -z "${OPENSHELL_VM_DRIVER_BIN:-}" ]; then mise run vm:setup fi - # Always rebuild the guest bundle so an e2e run cannot silently exercise a - # stale boundary binary after supervisor or isolation-interface changes. - echo "==> Building bundled VM supervisor (mise run vm:supervisor)" - mise run vm:supervisor + if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then + echo "==> Building bundled VM supervisor (mise run vm:supervisor)" + mise run vm:supervisor + fi export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR}}" else diff --git a/tasks/scripts/vm/build-supervisor-bundle.sh b/tasks/scripts/vm/build-supervisor-bundle.sh index a31e999603..0085c0619d 100755 --- a/tasks/scripts/vm/build-supervisor-bundle.sh +++ b/tasks/scripts/vm/build-supervisor-bundle.sh @@ -60,7 +60,6 @@ esac SUPERVISOR_BIN="${ROOT}/target/${RUST_TARGET}/release/openshell-sandbox" SUPERVISOR_OUTPUT="${OUTPUT_DIR}/openshell-sandbox.zst" -SUPERVISOR_RUNTIME_OUTPUT="${OUTPUT_DIR}/openshell-runtime.tar.zst" echo "==> Building openshell-sandbox supervisor bundle" echo " Guest arch: ${GUEST_ARCH}" @@ -124,51 +123,6 @@ fi zstd -19 -T0 -f "${SUPERVISOR_BIN}" -o "${SUPERVISOR_OUTPUT}" -case "${GUEST_ARCH}" in - aarch64|arm64) DOCKER_ARCH="arm64" ;; - x86_64|amd64) DOCKER_ARCH="amd64" ;; -esac - -echo "==> Building trusted supervisor helper runtime" -STAGED_SUPERVISOR="${ROOT}/deploy/docker/.build/prebuilt-binaries/${DOCKER_ARCH}/openshell-sandbox" -RUNTIME_IMAGE="openshell-vm-helper-runtime:${DOCKER_ARCH}-$$" -mkdir -p "$(dirname "${STAGED_SUPERVISOR}")" -cp "${SUPERVISOR_BIN}" "${STAGED_SUPERVISOR}" - -case "$(uname -m)" in - aarch64|arm64) HOST_DOCKER_ARCH="arm64" ;; - x86_64|amd64) HOST_DOCKER_ARCH="amd64" ;; - *) HOST_DOCKER_ARCH="" ;; -esac - -if [ "${HOST_DOCKER_ARCH}" = "${DOCKER_ARCH}" ]; then - docker build \ - --build-arg "TARGETARCH=${DOCKER_ARCH}" \ - --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ - --tag "${RUNTIME_IMAGE}" \ - "${ROOT}" -else - docker buildx build \ - --load \ - --platform "linux/${DOCKER_ARCH}" \ - --build-arg "TARGETARCH=${DOCKER_ARCH}" \ - --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ - --tag "${RUNTIME_IMAGE}" \ - "${ROOT}" -fi - -RUNTIME_CONTAINER="$(docker create "${RUNTIME_IMAGE}")" -cleanup_runtime_image() { - docker rm -f "${RUNTIME_CONTAINER}" >/dev/null 2>&1 || true - docker image rm "${RUNTIME_IMAGE}" >/dev/null 2>&1 || true -} -trap cleanup_runtime_image EXIT -docker cp "${RUNTIME_CONTAINER}:/openshell-runtime" - \ - | zstd -19 -T0 -f -o "${SUPERVISOR_RUNTIME_OUTPUT}" -cleanup_runtime_image -trap - EXIT - echo "==> Bundled supervisor ready" echo " Binary: $(du -sh "${SUPERVISOR_BIN}" | cut -f1)" echo " Compressed: $(du -sh "${SUPERVISOR_OUTPUT}" | cut -f1)" -echo " Helper runtime: $(du -sh "${SUPERVISOR_RUNTIME_OUTPUT}" | cut -f1)" From bfcd17868fe017913321069b17f82e36464fd0a4 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 06:06:16 -0700 Subject: [PATCH 33/37] fix(process): gate runtime CA helpers on Linux Signed-off-by: Drew Newberry --- crates/openshell-supervisor-process/src/process.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 2829f7bbf1..acebd742ec 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -163,6 +163,7 @@ pub(crate) fn prepare_child_sandbox( Ok(Some(prepared)) } +#[cfg(target_os = "linux")] fn policy_with_runtime_read_only( policy: &SandboxPolicy, runtime_read_only: &[PathBuf], @@ -176,6 +177,7 @@ fn policy_with_runtime_read_only( effective_policy } +#[cfg(target_os = "linux")] pub(crate) fn ca_runtime_read_only_paths(ca_paths: Option<&(PathBuf, PathBuf)>) -> Vec { let Some((certificate, bundle)) = ca_paths else { return Vec::new(); @@ -3491,6 +3493,7 @@ mod tests { } } + #[cfg(target_os = "linux")] #[test] fn runtime_ca_paths_are_added_to_the_effective_read_only_policy() { let mut policy = policy_with_process(ProcessPolicy::default()); From 699b4a9e9db4c81c6b36abc79971529a4ccfabe0 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 06:11:06 -0700 Subject: [PATCH 34/37] fix(sandbox): import control shutdown timeout portably Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 11d8e2fe83..77c85398e4 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -185,7 +185,6 @@ use openshell_supervisor_process::process::ProcessEnforcementMode; pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; use openshell_supervisor_process::skills; use tokio::sync::mpsc::UnboundedSender; -#[cfg(any(test, target_os = "linux"))] use tokio::time::timeout; const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; From 618579217ef5feeba1298683894efc168d803ab9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 06:22:16 -0700 Subject: [PATCH 35/37] fix(isolation): initialize TLS crypto provider Signed-off-by: Drew Newberry --- crates/openshell-isolation-interface/src/remote.rs | 1 + crates/openshell-sandbox/src/boundary_server.rs | 1 + crates/openshell-supervisor-network/src/l7/tls.rs | 1 + 3 files changed, 3 insertions(+) diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index f2fc128ceb..ce6cb0f8d6 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -177,6 +177,7 @@ fn validate_tcp_address(address: std::net::SocketAddr) -> Result<(), BackendErro } fn tls_client_config(ca_certificate_pem: &str) -> Result { + let _ = rustls::crypto::ring::default_provider().install_default(); let certificates = rustls_pemfile::certs(&mut ca_certificate_pem.as_bytes()) .collect::, _>>() .map_err(|error| { diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index e1f8bce0aa..98882d2624 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -1674,6 +1674,7 @@ mod linux { certificate_chain_path: &Path, private_key_path: &Path, ) -> io::Result { + let _ = rustls::crypto::ring::default_provider().install_default(); let certificate_bytes = std::fs::read(certificate_chain_path)?; let certificates = rustls_pemfile::certs(&mut certificate_bytes.as_slice()) .collect::, _>>()?; diff --git a/crates/openshell-supervisor-network/src/l7/tls.rs b/crates/openshell-supervisor-network/src/l7/tls.rs index 7c8301f8a7..4b8be4736e 100644 --- a/crates/openshell-supervisor-network/src/l7/tls.rs +++ b/crates/openshell-supervisor-network/src/l7/tls.rs @@ -100,6 +100,7 @@ impl SandboxCa { /// Load a durable CA while preserving the exact certificate bytes supplied /// by the provisioner for boundary launch replay. pub fn from_pem(certificate_pem: &str, private_key_pem: &str) -> Result { + let _ = rustls::crypto::ring::default_provider().install_default(); let ca_key = KeyPair::from_pem(private_key_pem) .into_diagnostic() .wrap_err("parse proxy CA private key")?; From 56cfb4ea38d33389fa9db5644e731acf3c820a7e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 06:31:56 -0700 Subject: [PATCH 36/37] test(isolation): initialize TLS provider per process Signed-off-by: Drew Newberry --- crates/openshell-isolation-interface/src/remote.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs index ce6cb0f8d6..1055f6bcb9 100644 --- a/crates/openshell-isolation-interface/src/remote.rs +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -1075,6 +1075,7 @@ mod tests { } fn test_certificate(name: &str) -> TestCertificate { + let _ = rustls::crypto::ring::default_provider().install_default(); let certified = generate_simple_self_signed(vec![name.to_string()]).expect("generate test certificate"); let certificate = certified.cert.der().clone(); From 35727e8ce98d2afd29b18c353002cb8c446ef90c Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 07:15:35 -0700 Subject: [PATCH 37/37] chore(stack): preserve managed child state Signed-off-by: Drew Newberry --- crates/openshell-supervisor-process/src/process.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index acebd742ec..b417440906 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -691,6 +691,8 @@ pub struct ProcessHandle { io: Option, terminal: Arc, signal_lock: Arc>, + #[cfg(target_os = "linux")] + managed_child: Option, } /// Supervisor-owned canonical-process I/O. These handles outlive individual @@ -971,6 +973,8 @@ impl ProcessHandle { io: Some(io), terminal: Arc::new(AtomicBool::new(false)), signal_lock: Arc::new(std::sync::Mutex::new(())), + #[cfg(target_os = "linux")] + managed_child, }) }