From 25881f10373255c21a4a412e512a8624fc35bc75 Mon Sep 17 00:00:00 2001 From: Drew Newberry <385+drew@users.noreply.github.com> Date: Wed, 26 Aug 2026 04:45:36 +0000 Subject: [PATCH 1/3] feat(isolation): add RFC 0012 backend contract Signed-off-by: Drew Newberry <385+drew@users.noreply.github.com> --- AGENTS.md | 1 + Cargo.lock | 9 + crates/openshell-isolation/BUILD.bazel | 30 + crates/openshell-isolation/Cargo.toml | 22 + crates/openshell-isolation/src/contract.rs | 609 +++++++++++++++ .../openshell-isolation/src/contract/tests.rs | 716 ++++++++++++++++++ crates/openshell-isolation/src/lib.rs | 50 ++ rfc/0012-isolation-backend/README.md | 394 ++++++++++ .../codebase-grounding.md | 26 + rfc/0012-isolation-backend/topology-matrix.md | 36 + 10 files changed, 1893 insertions(+) create mode 100644 crates/openshell-isolation/BUILD.bazel create mode 100644 crates/openshell-isolation/Cargo.toml create mode 100644 crates/openshell-isolation/src/contract.rs create mode 100644 crates/openshell-isolation/src/contract/tests.rs create mode 100644 crates/openshell-isolation/src/lib.rs create mode 100644 rfc/0012-isolation-backend/README.md create mode 100644 rfc/0012-isolation-backend/codebase-grounding.md create mode 100644 rfc/0012-isolation-backend/topology-matrix.md diff --git a/AGENTS.md b/AGENTS.md index 8c88c94814..604395353c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,6 +32,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-cli/` | CLI binary | User-facing command-line interface | | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary | | `crates/openshell-sandbox/` | Sandbox runtime | Container supervision, policy-enforced egress routing | +| `crates/openshell-isolation/` | Isolation backend contract | RFC 0012 `IsolationBackend` trait + types; the supervisor-facing runtime contract for the boundary | | `crates/openshell-policy/` | Policy engine | Filesystem, network, process, and inference constraints | | `crates/openshell-router/` | Privacy router | Privacy-aware LLM routing | | `crates/openshell-bootstrap/` | Gateway metadata | Gateway registration metadata, auth token storage, mTLS bundle storage | diff --git a/Cargo.lock b/Cargo.lock index 3ae582a12a..549b1fc834 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4147,6 +4147,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "openshell-isolation" +version = "0.0.0" +dependencies = [ + "async-trait", + "openshell-core", + "tokio", +] + [[package]] name = "openshell-ocsf" version = "0.0.0" diff --git a/crates/openshell-isolation/BUILD.bazel b/crates/openshell-isolation/BUILD.bazel new file mode 100644 index 0000000000..8bc2b233e3 --- /dev/null +++ b/crates/openshell-isolation/BUILD.bazel @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +load("@crates//:defs.bzl", "aliases", "all_crate_deps") +load("@rules_rs//rs:rust_library.bzl", "rust_library") +load("@rules_rs//rs:rust_test.bzl", "rust_test") +load("@rules_rust//rust:defs.bzl", "rustfmt_test") + +rust_library( + name = "openshell-isolation", + srcs = glob(["src/**/*.rs"]), + aliases = aliases(), + visibility = ["//visibility:public"], + deps = all_crate_deps(normal = True), +) + +rust_test( + name = "openshell-isolation_test", + crate = ":openshell-isolation", + deps = all_crate_deps(normal_dev = True), +) + +rustfmt_test( + name = "rustfmt_test", + targets = [ + ":openshell-isolation", + ":openshell-isolation_test", + ], + visibility = ["//crates:__pkg__"], +) diff --git a/crates/openshell-isolation/Cargo.toml b/crates/openshell-isolation/Cargo.toml new file mode 100644 index 0000000000..be703c52c4 --- /dev/null +++ b/crates/openshell-isolation/Cargo.toml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-isolation" +description = "OpenShell Isolation Backend runtime contract (RFC 0012)" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false } +async-trait = "0.1" +tokio = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-isolation/src/contract.rs b/crates/openshell-isolation/src/contract.rs new file mode 100644 index 0000000000..f94212ca9e --- /dev/null +++ b/crates/openshell-isolation/src/contract.rs @@ -0,0 +1,609 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Runtime-selectable Isolation Backend contract (RFC 0012). +//! +//! This module is the object-safe, runtime-selectable contract the supervisor +//! role drives. A backend registers an [`IsolationBackend`] under a +//! `backend_name`; the supervisor resolves it from a [`BackendRegistry`] +//! against the admitted backend name and advances the boundary through a fixed +//! chain of boxed states: +//! +//! ```text +//! attach topology + sandbox context -> Bound -> confirm -> Ready +//! -> start_agent -> Running +//! ``` +//! +//! Each transition consumes the prior state by value (`self: Box`), and no +//! state type has a public constructor, so a stage cannot be skipped or +//! replayed. The supervisor holds no `match`/downcast on concrete backends: the +//! registry is the only lookup by `backend_name`, and everything past it is a +//! `Box` / `Arc`. +//! +//! `attach` is atomic from the caller's perspective: it returns `Bound` or fails +//! closed, and it never binds a resource that is already bound to an active +//! boundary. Binary identity travels on every [`MediatedConnection`], resolved +//! by the backend for that exact connection; an unresolved identity denies the +//! connection and never authorizes anything. +//! +//! The contract is transport-neutral. Concrete topology implementations keep +//! their placement and coordination details behind these interfaces. + +use std::collections::HashMap; +use std::fmt; +use std::net::IpAddr; +use std::path::PathBuf; +use std::str::FromStr; +use std::sync::Arc; + +use async_trait::async_trait; +use tokio::io::{AsyncRead, AsyncWrite}; + +pub use openshell_core::policy::SandboxPolicy; + +/// The Isolation Backend contract version. The descriptor and the resolved +/// backend must both equal the supervisor-supported version exactly. +pub const INTERFACE_VERSION: u32 = 1; + +// ============================================================================ +// Errors +// ============================================================================ + +/// Classified failures at the common contract boundary. +/// +/// An error never advances the lifecycle or authorizes an operation. +#[derive(Debug)] +pub enum BackendError { + /// Descriptor missing, malformed, unsupported, or mismatched against admission. + Descriptor(String), + /// No backend registered for the resolved `backend_name`. + NotRegistered(String), + /// Authenticated attachment rejection (incompatible or already-bound resource). + Denied(String), + /// Boundary temporarily unavailable. + Unavailable(String), + /// Attachment-phase failure (establishment or mediation bring-up). + Attach(String), + /// Readiness confirmation failed (do not start workload code). + Confirm(String), + /// Process start or exec failure. + Process(String), + /// Abnormal boundary or workload loss, or an operation against an inactive + /// boundary. + Terminated(String), +} + +/// Coarse, machine-readable classification of a [`BackendError`] for supervisor +/// status mapping. The error's variant and message carry the structured context +/// (which operation failed). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackendErrorKind { + /// Descriptor, version, or backend mismatch. + Invalid, + /// Authenticated attachment rejection. + Denied, + /// Transient inability to serve an operation. + Unavailable, + /// Attachment, confirmation, start, or runtime operation failure. + Failed, + /// Abnormal boundary/workload loss, or an operation against an inactive + /// boundary. + Terminated, +} + +impl BackendError { + /// The machine-readable kind for this error. + #[must_use] + pub fn kind(&self) -> BackendErrorKind { + match self { + Self::Descriptor(_) | Self::NotRegistered(_) => BackendErrorKind::Invalid, + Self::Denied(_) => BackendErrorKind::Denied, + Self::Unavailable(_) => BackendErrorKind::Unavailable, + Self::Attach(_) | Self::Confirm(_) | Self::Process(_) => BackendErrorKind::Failed, + Self::Terminated(_) => BackendErrorKind::Terminated, + } + } +} + +impl fmt::Display for BackendError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Descriptor(m) => write!(f, "descriptor error: {m}"), + Self::NotRegistered(m) => write!(f, "backend not registered: {m}"), + Self::Denied(m) => write!(f, "attachment denied: {m}"), + Self::Unavailable(m) => write!(f, "boundary unavailable: {m}"), + Self::Attach(m) => write!(f, "attachment failed: {m}"), + Self::Confirm(m) => write!(f, "confirmation failed: {m}"), + Self::Process(m) => write!(f, "process error: {m}"), + Self::Terminated(m) => write!(f, "boundary terminated: {m}"), + } + } +} + +impl std::error::Error for BackendError {} + +/// Why an identity resolution failed. Resolution failure fails closed: the +/// mediation service denies and audits the connection; it never authorizes. +#[derive(Debug, Clone)] +pub enum ResolveError { + /// No process owns the connection (stale or unknown attribution). + NotFound, + /// Resolution attempted but could not produce trustworthy identity. + Failed(String), +} + +impl fmt::Display for ResolveError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotFound => write!(f, "connection owner not found"), + Self::Failed(m) => write!(f, "identity resolution failed: {m}"), + } + } +} + +impl std::error::Error for ResolveError {} + +// ============================================================================ +// Descriptor and registry +// ============================================================================ + +/// The common topology descriptor envelope. +/// +/// The compute driver supplies one for every provisioned topology, including +/// resources prepared before sandbox assignment. The opaque payload identifies, +/// or gives the backend enough information to resolve, the exact +/// driver-provisioned resource; its protection is backend-specific. +#[derive(Debug, Clone)] +pub struct TopologyDescriptor { + /// The Isolation Backend contract version this descriptor targets. + pub version: u32, + /// The backend the supervisor must instantiate. + pub backend_name: String, + /// Backend-specific attachment data. + pub payload: Vec, +} + +/// A descriptor whose common envelope has passed registry verification. +/// +/// Minted only by [`BackendRegistry::resolve`]; no public constructor, so an +/// unverified descriptor cannot reach a backend. The type does not imply that +/// the opaque payload has been validated: the backend validates the payload and +/// atomically binds it to the sandbox context during `attach`. +pub struct VerifiedTopologyDescriptor { + descriptor: TopologyDescriptor, +} + +impl VerifiedTopologyDescriptor { + /// The verified backend name. + #[must_use] + pub fn backend_name(&self) -> &str { + &self.descriptor.backend_name + } + /// The backend-specific payload (validated by the backend at `attach`). + #[must_use] + pub fn payload(&self) -> &[u8] { + &self.descriptor.payload + } + /// The interface version. + #[must_use] + pub fn version(&self) -> u32 { + self.descriptor.version + } +} + +/// The trusted sandbox context, constructed by trusted common code after the +/// control plane assigns the resource to the admitted sandbox. +/// +/// Carries the admitted create-time policy. Approved network-policy revisions +/// are made effective by supervisor-owned network mediation, outside the +/// backend lifecycle. +pub struct SandboxContext { + /// Which sandbox this is. + pub sandbox_id: String, + /// The admitted create-time policy. + pub policy: SandboxPolicy, + /// The admitted agent workload. + pub agent: AgentSpec, +} + +/// The agent workload to run inside the boundary. +pub use crate::AgentSpec; + +/// Maps backend name to its implementation. This is the only lookup by name; +/// supervisor lifecycle never branches on a concrete backend, and resolution +/// never falls back to another backend. +#[derive(Default)] +pub struct BackendRegistry { + backends: HashMap>, +} + +impl BackendRegistry { + /// An empty registry. + #[must_use] + pub fn new() -> Self { + Self { + backends: HashMap::new(), + } + } + + /// Register a backend. Rejects a duplicate name or a backend that does not + /// speak the supervisor-supported interface version exactly. + /// + /// # Errors + /// + /// Returns [`BackendError::Descriptor`] for a duplicate `backend_name` or + /// an interface-version mismatch. + pub fn register(&mut self, backend: Arc) -> Result<(), BackendError> { + let name = backend.backend_name().to_string(); + if self.backends.contains_key(&name) { + return Err(BackendError::Descriptor(format!( + "duplicate backend name {name:?}" + ))); + } + if backend.version() != INTERFACE_VERSION { + return Err(BackendError::Descriptor(format!( + "backend {name:?} targets interface version {}, supervisor speaks {INTERFACE_VERSION}", + backend.version() + ))); + } + self.backends.insert(name, backend); + Ok(()) + } + + /// Verify the descriptor's common envelope against the admitted backend name + /// and resolve its backend. Fails closed and never falls back: + /// + /// - the descriptor's interface version must equal [`INTERFACE_VERSION`]; + /// - the descriptor's `backend_name` must equal the admitted name; + /// - a backend must be registered under that name; and + /// - the backend's version must equal [`INTERFACE_VERSION`] exactly. + /// + /// # Errors + /// + /// Returns [`BackendError::Descriptor`] for a version or admission + /// mismatch, and [`BackendError::NotRegistered`] when no backend is + /// registered for the admitted name. + pub fn resolve( + &self, + descriptor: TopologyDescriptor, + admitted_backend_name: &str, + ) -> Result<(Arc, VerifiedTopologyDescriptor), BackendError> { + if descriptor.version != INTERFACE_VERSION { + return Err(BackendError::Descriptor(format!( + "descriptor interface version {} unsupported (expected {INTERFACE_VERSION})", + descriptor.version + ))); + } + if descriptor.backend_name != admitted_backend_name { + return Err(BackendError::Descriptor(format!( + "descriptor backend {:?} does not match admitted backend {admitted_backend_name:?}", + descriptor.backend_name + ))); + } + let backend = self + .backends + .get(&descriptor.backend_name) + .ok_or_else(|| BackendError::NotRegistered(descriptor.backend_name.clone()))? + .clone(); + if backend.backend_name() != descriptor.backend_name { + return Err(BackendError::Descriptor(format!( + "registry returned backend {:?} for name {:?}", + backend.backend_name(), + descriptor.backend_name + ))); + } + if backend.version() != INTERFACE_VERSION { + return Err(BackendError::Descriptor(format!( + "backend {:?} speaks interface version {}, supervisor requires {INTERFACE_VERSION}", + descriptor.backend_name, + backend.version() + ))); + } + Ok((backend, VerifiedTopologyDescriptor { descriptor })) + } +} + +/// Establishes and operates boundaries for one admitted backend implementation. +#[async_trait] +pub trait IsolationBackend: Send + Sync { + /// The stable registered backend name. + fn backend_name(&self) -> &str; + + /// The Isolation Backend contract version this backend speaks. Matched + /// exactly against [`INTERFACE_VERSION`]; there is no capability negotiation. + fn version(&self) -> u32; + + /// Validate the opaque payload and atomically bind it to the trusted + /// sandbox context: returns `Bound` or fails closed. Never binds a resource + /// that is already bound to an active boundary. + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError>; +} + +// ============================================================================ +// Lifecycle states +// ============================================================================ + +/// Bound: the topology descriptor and trusted sandbox context are bound to the +/// same resource, and the mediation source is available. No untrusted workload +/// code is running. +#[async_trait] +pub trait BoundBoundary: Send { + /// The mediation service's backend-neutral source of workload connections. + /// Retained by the supervisor before consuming `Bound`. + fn network_mediation_source(&self) -> Arc; + + /// Confirm standing enforcement. How a backend establishes confidence is + /// private to that backend; confirmation fails closed. + async fn confirm(self: Box) -> Result, BackendError>; +} + +/// Ready: standing enforcement is confirmed, and the backend is prepared to +/// ensure the admitted launch-time controls are in force +/// before untrusted execution. Only agent activation is possible from here. +#[async_trait] +pub trait ReadyBoundary: Send { + /// Make the admitted agent runnable behind the boundary and return its + /// handle. `start_agent` is the sole operation that may make the admitted + /// agent runnable, and it fails closed if any `Ready` condition no longer + /// holds. Whether the backend creates the agent process or releases a held, + /// driver-provisioned execution object is backend-specific; every + /// applicable launch-time control is in force before the first untrusted + /// instruction. + async fn start_agent(self: Box) -> Result, BackendError>; +} + +/// Running: the agent is runnable behind the boundary and the returned agent +/// handle represents the admitted agent process. Exec and forwarding are available. +/// +/// All interface accessors return owned `Arc`s so a consumer can retain them +/// past any later state consumption. +pub trait RunningBoundary: Send + Sync { + /// The admitted agent process handle. + fn agent(&self) -> Arc; + /// The in-boundary exec interface. + fn exec(&self) -> Arc; + /// The loopback port-forward interface. + fn port_forward(&self) -> Arc; +} + +// ============================================================================ +// Process and exec +// ============================================================================ + +/// Placement-neutral terminal status of a boundary process. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BoundaryExitStatus { + /// Exited with a code. + Exited(i32), + /// Killed by a signal. + Signaled(i32), +} + +/// Placement-neutral signal to deliver to a boundary process. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BoundarySignal { + /// Graceful terminate. + Term, + /// Forceful kill. + Kill, + /// Interrupt. + Int, + /// Hangup. + Hup, +} + +/// A process running inside the boundary. `wait` returns one stable status +/// however many times it is called; a local PID is never the process handle. +#[async_trait] +pub trait BoundaryProcess: Send + Sync { + /// Await terminal status (stable across repeated calls). + async fn wait(&self) -> Result; + /// Deliver a signal to the process or its group. + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError>; + /// Terminate the process and its backend-owned process group. + async fn terminate(&self) -> Result<(), BackendError>; +} + +/// A boxed async writer into a boundary process's stdin. +pub type BoundaryInput = Box; +/// A boxed async reader from a boundary process's stdout or stderr. +pub type BoundaryOutput = Box; + +/// A PTY attached to an exec session. +#[async_trait] +pub trait BoundaryTerminal: Send + Sync { + /// Resize the terminal. + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError>; +} + +/// An owned exec session: the process handle plus its stdio and optional PTY. +/// Owning the process keeps it alive after `exec` returns. +pub struct ExecSession { + /// The spawned process. + pub process: Arc, + /// Stdin writer, if not a PTY-merged stream. + pub stdin: Option, + /// Stdout reader. + pub stdout: BoundaryOutput, + /// Stderr reader, distinct from stdout for non-PTY exec. + pub stderr: Option, + /// PTY control, present when a terminal was requested. + pub terminal: Option>, +} + +/// What to run inside the boundary via [`BoundaryExec`]. +#[derive(Debug, Clone)] +pub struct ExecSpec { + /// Program to run. + pub program: String, + /// Program arguments. + pub args: Vec, + /// Extra environment over the boundary's base. + pub env: Vec<(String, String)>, + /// Working directory, if any. + pub workdir: Option, + /// Whether to allocate a PTY. + pub pty: bool, +} + +/// In-boundary process entry, consumed by the SSH server and supervisor session. +/// +/// Like `start_agent`, every exec ensures the applicable launch-time controls +/// are in force before the new process executes its first untrusted instruction +/// and preserves the provisioned execution environment. +#[async_trait] +pub trait BoundaryExec: Send + Sync { + /// Spawn `spec` inside the boundary, returning an owned session. + async fn exec(&self, spec: ExecSpec) -> Result; +} + +// ============================================================================ +// Port forward +// ============================================================================ + +/// A loopback-only target inside the boundary, validated at construction. +#[derive(Debug, Clone)] +pub struct LoopbackTarget { + host: IpAddr, + port: u16, +} + +impl LoopbackTarget { + /// Build a loopback target, rejecting any non-loopback host. + /// + /// # Errors + /// + /// Returns [`BackendError::Process`] when `host` is not a loopback address. + pub fn new(host: IpAddr, port: u16) -> Result { + if !host.is_loopback() { + return Err(BackendError::Process(format!( + "port-forward target {host} is not loopback" + ))); + } + Ok(Self { host, port }) + } + /// The loopback host. + #[must_use] + pub fn host(&self) -> IpAddr { + self.host + } + /// The target port. + #[must_use] + pub fn port(&self) -> u16 { + self.port + } +} + +/// A bidirectional byte stream into the boundary. +pub trait DuplexStream: AsyncRead + AsyncWrite + Send + Unpin {} +impl DuplexStream for T {} + +/// An open connection into a boundary loopback target. +pub type BoundaryDuplexStream = Box; + +/// Loopback port-forward, consumed by the SSH server and supervisor session. +#[async_trait] +pub trait BoundaryPortForward: Send + Sync { + /// Connect to `target` inside the boundary. + async fn connect(&self, target: LoopbackTarget) -> Result; +} + +// ============================================================================ +// Mediation and binary identity +// ============================================================================ + +/// Executable identity for one accepted connection, resolved by the backend and +/// delivered on [`MediatedConnection`] before the mediation service evaluates +/// policy. +/// +/// A missing digest is `None`, never an empty value; policy that requires an +/// unavailable identity field cannot authorize the connection. How a backend +/// resolves identity is private to that backend; the shape and the fail-closed +/// semantics do not change. +#[derive(Debug, Clone)] +pub struct BinaryIdentity { + /// Absolute path of the executable resolved for the accepted connection. + pub binary_path: PathBuf, + /// Digest of the resolved executable object. `None` when unavailable. + pub binary_digest: Option, + /// Ancestor process binaries, nearest first. + pub ancestors: Vec, + /// Absolute script/interpreter paths drawn from the process cmdlines. + /// Diagnostic context; never authorizes. + pub cmdline_paths: Vec, +} + +/// A SHA-256 digest, kept typed so the identity field is not coupled to its +/// textual encoding or forced to repeat the algorithm in its name. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Sha256Digest([u8; 32]); + +impl Sha256Digest { + /// Return the raw digest bytes. + #[must_use] + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +impl fmt::Display for Sha256Digest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for byte in self.0 { + write!(f, "{byte:02x}")?; + } + Ok(()) + } +} + +impl FromStr for Sha256Digest { + type Err = ResolveError; + + fn from_str(value: &str) -> Result { + if value.len() != 64 || !value.is_ascii() { + return Err(ResolveError::Failed( + "SHA-256 digest must contain 64 hexadecimal characters".to_string(), + )); + } + let mut bytes = [0_u8; 32]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).map_err(|_| { + ResolveError::Failed("SHA-256 digest contains non-hexadecimal data".to_string()) + })?; + } + Ok(Self(bytes)) + } +} + +/// A workload connection delivered to the mediation service, carrying the +/// identity-resolution result for that connection. +/// +/// An `Err` identity denies the connection and is audited; it never authorizes +/// anything. +pub struct MediatedConnection { + /// The workload connection stream. + pub stream: BoundaryDuplexStream, + /// Executable identity, resolved by the backend for this connection. + pub binary_identity: Result, +} + +/// A logical per-boundary stream of workload connections, consumed by the +/// mediation service wherever that service runs. +/// +/// It may wrap a dedicated listener or a demultiplexed view over shared +/// transport; how it reaches a co-located proxy, a sidecar, or a shared +/// mediation service is backend-private. A trusted backend component associates +/// every returned connection with its active boundary without relying solely on +/// a transport tuple or workload-provided identifier. An `Err` from `accept` +/// means the source itself is unusable and fails the boundary closed. +#[async_trait] +pub trait NetworkMediationSource: Send + Sync { + /// Await the next mediated workload connection. + async fn accept(&self) -> Result; +} + +#[cfg(test)] +mod tests; diff --git a/crates/openshell-isolation/src/contract/tests.rs b/crates/openshell-isolation/src/contract/tests.rs new file mode 100644 index 0000000000..645bb7edee --- /dev/null +++ b/crates/openshell-isolation/src/contract/tests.rs @@ -0,0 +1,716 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Conformance harness for the runtime-selectable contract. +//! +//! Two materially different mock backends (`Primary`, `Secondary`) with +//! distinct concrete state structs (each generic over a marker, so each kind +//! monomorphizes to its own types) prove the registry holds heterogeneous +//! backends behind `dyn` with no enum over concrete state, and that one driver +//! runs both unchanged. + +use std::marker::PhantomData; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; + +use super::*; + +// --------------------------------------------------------------------------- +// Marker kinds: two materially different backends. +// --------------------------------------------------------------------------- + +trait MockKind: Send + Sync + 'static { + const BACKEND_ID: &'static str; + /// Whether this backend can produce a binary digest (a heterogeneity axis: + /// one backend resolves a full identity, the other resolves path-only). + const HAS_DIGEST: bool; +} + +struct Primary; +impl MockKind for Primary { + const BACKEND_ID: &'static str = "mock-primary"; + const HAS_DIGEST: bool = true; +} + +struct Secondary; +impl MockKind for Secondary { + const BACKEND_ID: &'static str = "mock-secondary"; + const HAS_DIGEST: bool = false; +} + +// --------------------------------------------------------------------------- +// Runtime interfaces (shared across kinds where behavior is identical). +// --------------------------------------------------------------------------- + +struct MockProcess { + status: BoundaryExitStatus, + alive: AtomicBool, + signals: Mutex>, +} + +impl MockProcess { + fn new() -> Arc { + Arc::new(Self { + status: BoundaryExitStatus::Exited(0), + alive: AtomicBool::new(true), + signals: Mutex::new(Vec::new()), + }) + } +} + +#[async_trait] +impl BoundaryProcess for MockProcess { + async fn wait(&self) -> Result { + // Stable across repeated calls. + self.alive.store(false, Ordering::SeqCst); + Ok(self.status) + } + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + if !self.alive.load(Ordering::SeqCst) { + return Err(BackendError::Terminated("process has exited".to_string())); + } + self.signals.lock().unwrap().push(signal); + Ok(()) + } + async fn terminate(&self) -> Result<(), BackendError> { + self.alive + .swap(false, Ordering::SeqCst) + .then_some(()) + .ok_or_else(|| BackendError::Terminated("process has exited".to_string())) + } +} + +/// Mediation source: hands the mediation service a connection carrying its +/// per-connection identity-resolution result. +struct MockSource(PhantomData); + +#[async_trait] +impl NetworkMediationSource for MockSource { + async fn accept(&self) -> Result { + let (near, _far) = tokio::io::duplex(64); + Ok(MediatedConnection { + stream: Box::new(near), + binary_identity: Ok(BinaryIdentity { + binary_path: PathBuf::from("/usr/bin/agent"), + binary_digest: K::HAS_DIGEST + .then(|| "00".repeat(32).parse().expect("valid digest")), + ancestors: vec![], + cmdline_paths: vec![], + }), + }) + } +} + +/// An source whose backend cannot attribute the connection: the connection is +/// still delivered, carrying `Err`, so the mediation service denies and audits +/// it. It never authorizes anything. +struct UnattributedSource; + +#[async_trait] +impl NetworkMediationSource for UnattributedSource { + async fn accept(&self) -> Result { + let (near, _far) = tokio::io::duplex(64); + Ok(MediatedConnection { + stream: Box::new(near), + binary_identity: Err(ResolveError::Failed("hash unavailable".to_string())), + }) + } +} + +struct MockExec; + +struct MockTerminal { + size: Mutex>, +} + +#[async_trait] +impl BoundaryTerminal for MockTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + *self.size.lock().unwrap() = Some((cols, rows)); + Ok(()) + } +} + +#[async_trait] +impl BoundaryExec for MockExec { + async fn exec(&self, spec: ExecSpec) -> Result { + let (_near, far) = tokio::io::duplex(64); + let (out_r, _out_w) = tokio::io::duplex(64); + let (err_r, _err_w) = tokio::io::duplex(64); + let stdin: BoundaryInput = Box::new(far); + let stderr: BoundaryOutput = Box::new(err_r); + let terminal: Arc = Arc::new(MockTerminal { + size: Mutex::new(None), + }); + Ok(ExecSession { + process: MockProcess::new(), + stdin: (!spec.pty).then_some(stdin), + stdout: Box::new(out_r), + stderr: (!spec.pty).then_some(stderr), + terminal: spec.pty.then_some(terminal), + }) + } +} + +struct MockPortForward; + +#[async_trait] +impl BoundaryPortForward for MockPortForward { + async fn connect(&self, _target: LoopbackTarget) -> Result { + let (near, far) = tokio::io::duplex(64); + tokio::spawn(async move { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut far = far; + let mut buf = [0u8; 4]; + if far.read_exact(&mut buf).await.is_ok() { + let _ = far.write_all(&buf).await; + } + }); + Ok(Box::new(near)) + } +} + +// --------------------------------------------------------------------------- +// Boxed lifecycle states (distinct concrete struct per kind). +// --------------------------------------------------------------------------- + +struct MockBound { + source: Arc>, +} +struct MockReady { + _k: PhantomData, +} +struct MockRunning { + process: Arc, + exec: Arc, + port_forward: Arc, + _k: PhantomData, +} + +#[async_trait] +impl BoundBoundary for MockBound { + fn network_mediation_source(&self) -> Arc { + self.source.clone() + } + async fn confirm(self: Box) -> Result, BackendError> { + Ok(Box::new(MockReady:: { _k: PhantomData })) + } +} + +#[async_trait] +impl ReadyBoundary for MockReady { + async fn start_agent(self: Box) -> Result, BackendError> { + Ok(Box::new(MockRunning:: { + process: MockProcess::new(), + exec: Arc::new(MockExec), + port_forward: Arc::new(MockPortForward), + _k: PhantomData, + })) + } +} + +impl RunningBoundary for MockRunning { + fn agent(&self) -> Arc { + self.process.clone() + } + fn exec(&self) -> Arc { + self.exec.clone() + } + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +/// One backend per boundary resource: `attach` is atomic and never binds a +/// resource that is already bound to an active boundary, so a second attach +/// against the same mock resource is `Denied`. +struct MockBackend { + attached: AtomicBool, + _k: PhantomData, +} + +impl MockBackend { + fn new() -> Self { + Self { + attached: AtomicBool::new(false), + _k: PhantomData, + } + } +} + +#[async_trait] +impl IsolationBackend for MockBackend { + fn backend_name(&self) -> &'static str { + K::BACKEND_ID + } + fn version(&self) -> u32 { + INTERFACE_VERSION + } + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + assert_eq!(descriptor.backend_name(), K::BACKEND_ID); + assert!(!sandbox.sandbox_id.is_empty()); + if self.attached.swap(true, Ordering::SeqCst) { + return Err(BackendError::Denied( + "resource is already bound to an active boundary".to_string(), + )); + } + Ok(Box::new(MockBound:: { + source: Arc::new(MockSource(PhantomData)), + })) + } +} + +/// A backend that speaks the wrong contract version; registration must reject it. +struct WrongVersionBackend; + +#[async_trait] +impl IsolationBackend for WrongVersionBackend { + fn backend_name(&self) -> &'static str { + "mock-wrong-version" + } + fn version(&self) -> u32 { + INTERFACE_VERSION + 1 + } + async fn attach( + &self, + _descriptor: VerifiedTopologyDescriptor, + _sandbox: SandboxContext, + ) -> Result, BackendError> { + unreachable!("must never be resolved") + } +} + +// --------------------------------------------------------------------------- +// Helpers. +// --------------------------------------------------------------------------- + +fn registry() -> BackendRegistry { + let mut reg = BackendRegistry::new(); + reg.register(Arc::new(MockBackend::::new())) + .expect("register primary"); + reg.register(Arc::new(MockBackend::::new())) + .expect("register secondary"); + reg +} + +fn descriptor(backend_name: &str) -> TopologyDescriptor { + TopologyDescriptor { + version: INTERFACE_VERSION, + backend_name: backend_name.to_string(), + payload: vec![], + } +} + +fn sandbox_ctx() -> SandboxContext { + SandboxContext { + sandbox_id: "sb-1".to_string(), + policy: SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }, + agent: AgentSpec { + program: "/bin/true".to_string(), + args: vec![], + workdir: None, + timeout_secs: 0, + interactive: false, + }, + } +} + +/// The backend-independent supervisor sequence. Identical for every backend: +/// this is the proof that adding a backend needs no supervisor lifecycle change. +async fn drive( + reg: &BackendRegistry, + descriptor: TopologyDescriptor, + admitted: &str, +) -> Result, BackendError> { + let (backend, verified) = reg.resolve(descriptor, admitted)?; + let bound = backend.attach(verified, sandbox_ctx()).await?; + // The mediation source is retained before consuming `Bound` and stays + // usable across the confirm/start transitions. + let _ingress = bound.network_mediation_source(); + let ready = bound.confirm().await?; + ready.start_agent().await +} + +// --------------------------------------------------------------------------- +// Registry and descriptor. +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn registry_selects_correct_backend() { + let reg = registry(); + let (f, _v) = reg + .resolve(descriptor("mock-secondary"), "mock-secondary") + .expect("resolve"); + assert_eq!(f.backend_name(), "mock-secondary"); +} + +#[test] +fn registry_rejects_duplicate_registration() { + let mut reg = BackendRegistry::new(); + reg.register(Arc::new(MockBackend::::new())) + .expect("first"); + let err = reg + .register(Arc::new(MockBackend::::new())) + .expect_err("duplicate must fail"); + assert!(matches!(err, BackendError::Descriptor(_))); +} + +#[test] +fn registry_rejects_wrong_backend_version() { + let mut reg = BackendRegistry::new(); + let err = reg + .register(Arc::new(WrongVersionBackend)) + .expect_err("wrong version must fail"); + assert_eq!(err.kind(), BackendErrorKind::Invalid); +} + +#[test] +fn registry_rejects_unknown_backend() { + let reg = registry(); + let err = reg + .resolve(descriptor("nope"), "nope") + .map(|_| ()) + .expect_err("unknown must fail"); + assert!(matches!(err, BackendError::NotRegistered(_))); +} + +#[test] +fn registry_rejects_descriptor_admission_mismatch_without_fallback() { + let reg = registry(); + // Descriptor names primary, admission says secondary: must fail, and must + // not silently fall back to either backend. + let err = reg + .resolve(descriptor("mock-primary"), "mock-secondary") + .map(|_| ()) + .expect_err("mismatch must fail"); + assert!(matches!(err, BackendError::Descriptor(_))); +} + +#[test] +fn registry_rejects_unsupported_version() { + let reg = registry(); + let mut d = descriptor("mock-primary"); + d.version = INTERFACE_VERSION + 1; + let err = reg + .resolve(d, "mock-primary") + .map(|_| ()) + .expect_err("bad version must fail"); + assert!(matches!(err, BackendError::Descriptor(_))); +} + +// --------------------------------------------------------------------------- +// Lifecycle: one driver, two heterogeneous backends, no consumer change. +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn one_driver_runs_both_backends() { + let reg = registry(); + // The exact same driver code runs a backend with distinct concrete state + // structs; the registry holds them behind `dyn`, no enum. + let primary = drive(®, descriptor("mock-primary"), "mock-primary") + .await + .expect("primary lifecycle"); + let secondary = drive(®, descriptor("mock-secondary"), "mock-secondary") + .await + .expect("secondary lifecycle"); + + // Both expose a usable agent process handle past start_agent. + assert_eq!( + primary.agent().wait().await.expect("wait"), + BoundaryExitStatus::Exited(0) + ); + assert_eq!( + secondary.agent().wait().await.expect("wait"), + BoundaryExitStatus::Exited(0) + ); +} + +#[tokio::test] +async fn one_boundary_termination_does_not_change_another_boundary() { + let reg = registry(); + let primary = drive(®, descriptor("mock-primary"), "mock-primary") + .await + .expect("primary lifecycle"); + let secondary = drive(®, descriptor("mock-secondary"), "mock-secondary") + .await + .expect("secondary lifecycle"); + + primary + .agent() + .terminate() + .await + .expect("terminate primary"); + secondary + .agent() + .signal(BoundarySignal::Term) + .await + .expect("secondary remains active"); +} + +#[tokio::test] +async fn attach_never_binds_an_already_bound_resource() { + let reg = registry(); + // First attach binds the mock resource. + drive(®, descriptor("mock-primary"), "mock-primary") + .await + .expect("first lifecycle"); + // A second attach against the same active boundary must be denied, not + // silently create a second binding. + let err = drive(®, descriptor("mock-primary"), "mock-primary") + .await + .map(|_| ()) + .expect_err("second attach must fail"); + assert_eq!(err.kind(), BackendErrorKind::Denied); +} + +#[tokio::test] +async fn runtime_interfaces_survive_lifecycle_consumption() { + let reg = registry(); + let (backend, verified) = reg + .resolve(descriptor("mock-primary"), "mock-primary") + .expect("resolve"); + let bound = backend + .attach(verified, sandbox_ctx()) + .await + .expect("attach"); + + // Retain the source at Bound, then consume the bound state with confirm. + // The retained Arc must remain usable afterward. + let source = bound.network_mediation_source(); + let ready = bound.confirm().await.expect("confirm"); + let _running = ready.start_agent().await.expect("start"); + + let conn = source.accept().await.expect("accept after consumption"); + let identity = conn.binary_identity.expect("identity resolves"); + assert_eq!(identity.binary_path, PathBuf::from("/usr/bin/agent")); +} + +// --------------------------------------------------------------------------- +// Process and I/O. +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn agent_process_survives_and_wait_is_stable() { + let reg = registry(); + let running = drive(®, descriptor("mock-primary"), "mock-primary") + .await + .expect("lifecycle"); + let agent = running.agent(); + // Survives start_agent returning; wait is stable across repeated calls. + assert_eq!( + agent.wait().await.expect("wait 1"), + BoundaryExitStatus::Exited(0) + ); + assert_eq!( + agent.wait().await.expect("wait 2"), + BoundaryExitStatus::Exited(0) + ); + assert!(matches!( + agent.signal(BoundarySignal::Term).await, + Err(BackendError::Terminated(_)) + )); +} + +#[tokio::test] +async fn every_signal_reaches_the_backend_unchanged() { + let process = MockProcess::new(); + for signal in [ + BoundarySignal::Term, + BoundarySignal::Kill, + BoundarySignal::Int, + BoundarySignal::Hup, + ] { + process.signal(signal).await.expect("signal"); + } + assert_eq!( + *process.signals.lock().unwrap(), + vec![ + BoundarySignal::Term, + BoundarySignal::Kill, + BoundarySignal::Int, + BoundarySignal::Hup, + ] + ); +} + +#[tokio::test] +async fn normal_and_signaled_exit_are_distinct_and_stable() { + let signaled = MockProcess { + status: BoundaryExitStatus::Signaled(9), + alive: AtomicBool::new(false), + signals: Mutex::new(Vec::new()), + }; + assert_eq!( + signaled.wait().await.expect("first wait"), + BoundaryExitStatus::Signaled(9) + ); + assert_eq!( + signaled.wait().await.expect("second wait"), + BoundaryExitStatus::Signaled(9) + ); + assert_ne!( + signaled.wait().await.expect("third wait"), + BoundaryExitStatus::Exited(137) + ); +} + +#[tokio::test] +async fn exec_session_owns_its_process_and_streams() { + let reg = registry(); + let running = drive(®, descriptor("mock-primary"), "mock-primary") + .await + .expect("lifecycle"); + let session = running + .exec() + .exec(ExecSpec { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), "true".to_string()], + env: vec![], + workdir: None, + pty: false, + }) + .await + .expect("exec"); + // The exec'd process survives `exec` returning, and stdout/stderr are distinct. + assert!(session.stderr.is_some()); + assert!(session.stdin.is_some()); + assert_eq!( + session.process.wait().await.expect("exec wait"), + BoundaryExitStatus::Exited(0) + ); +} + +#[tokio::test] +async fn pty_exec_merges_output_and_supports_resize() { + let session = MockExec + .exec(ExecSpec { + program: "/bin/sh".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: true, + }) + .await + .expect("pty exec"); + assert!(session.stdin.is_none()); + assert!(session.stderr.is_none()); + session + .terminal + .expect("terminal") + .resize(120, 40) + .await + .expect("resize"); +} + +#[tokio::test] +async fn port_forward_rejects_non_loopback() { + let target = LoopbackTarget::new("8.8.8.8".parse().unwrap(), 53); + assert!(target.is_err()); + let loopback = LoopbackTarget::new("127.0.0.1".parse().unwrap(), 8080).expect("loopback ok"); + assert_eq!(loopback.port(), 8080); +} + +#[tokio::test] +async fn validated_port_forward_stream_remains_usable() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let target = LoopbackTarget::new("127.0.0.1".parse().unwrap(), 8080).unwrap(); + let mut stream = MockPortForward.connect(target).await.expect("connect"); + stream.write_all(b"ping").await.expect("write"); + let mut response = [0_u8; 4]; + stream.read_exact(&mut response).await.expect("read"); + assert_eq!(&response, b"ping"); +} + +// --------------------------------------------------------------------------- +// Mediation and binary identity. +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn mediated_connection_carries_identity_for_that_connection() { + let reg = registry(); + let (backend, verified) = reg + .resolve(descriptor("mock-primary"), "mock-primary") + .expect("resolve"); + let bound = backend + .attach(verified, sandbox_ctx()) + .await + .expect("attach"); + let conn = bound + .network_mediation_source() + .accept() + .await + .expect("accept"); + let identity = conn.binary_identity.expect("identity resolves"); + assert_eq!(identity.binary_path, PathBuf::from("/usr/bin/agent")); + // A missing digest is `None`, never an empty value. + assert_eq!( + identity.binary_digest.expect("digest").to_string(), + "00".repeat(32) + ); +} + +#[tokio::test] +async fn missing_digest_is_none_never_empty() { + // The secondary backend resolves path-only identity: the digest is `None`, + // so policy that requires a digest cannot authorize the connection. + let source = MockSource::(PhantomData); + let conn = source.accept().await.expect("accept"); + let identity = conn.binary_identity.expect("identity resolves"); + assert!(identity.binary_digest.is_none()); +} + +#[tokio::test] +async fn unresolved_identity_travels_with_the_connection_and_fails_closed() { + // Attribution failure does not tear down the source: the connection is + // delivered carrying `Err`, and the mediation service denies it. + let source = UnattributedSource; + let conn = source.accept().await.expect("accept"); + assert!(conn.binary_identity.is_err()); +} + +// --------------------------------------------------------------------------- +// Errors. +// --------------------------------------------------------------------------- + +#[test] +fn error_kinds_map_to_supervisor_status_classes() { + assert_eq!( + BackendError::Descriptor("x".into()).kind(), + BackendErrorKind::Invalid + ); + assert_eq!( + BackendError::NotRegistered("x".into()).kind(), + BackendErrorKind::Invalid + ); + assert_eq!( + BackendError::Denied("x".into()).kind(), + BackendErrorKind::Denied + ); + assert_eq!( + BackendError::Unavailable("x".into()).kind(), + BackendErrorKind::Unavailable + ); + assert_eq!( + BackendError::Attach("x".into()).kind(), + BackendErrorKind::Failed + ); + assert_eq!( + BackendError::Confirm("x".into()).kind(), + BackendErrorKind::Failed + ); + assert_eq!( + BackendError::Terminated("x".into()).kind(), + BackendErrorKind::Terminated + ); +} diff --git a/crates/openshell-isolation/src/lib.rs b/crates/openshell-isolation/src/lib.rs new file mode 100644 index 0000000000..d7f4e32183 --- /dev/null +++ b/crates/openshell-isolation/src/lib.rs @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The `OpenShell` **Isolation Backend** runtime contract (RFC 0012). +//! +//! An isolation backend establishes and enforces a workload's isolation boundary; +//! the supervisor role drives it through one contract. The supervisor-facing +//! contract lives in [`contract`]: an object-safe, runtime-selectable backend +//! plus a fixed chain of boxed lifecycle states the supervisor advances without +//! branching on where the boundary sits. The same calls work whether the +//! boundary lives in the agent's container (the in-pod backend) or further out +//! (a microVM, a node daemon, a separate pod). +//! +//! The backend establishes standing enforcement before untrusted code runs and +//! ensures launch-time controls are in force before each process's first +//! untrusted instruction. It also exposes process operations and supplies +//! workload egress to supervisor-owned network mediation. +//! +//! # Ordering is a security property +//! +//! The lifecycle states run in order: attach -> Bound -> confirm -> Ready -> +//! `start_agent` -> Running. Nothing untrusted runs inside the boundary until it +//! is confirmed ready. This is enforced *by construction*: each transition +//! consumes the prior state by value, and no state type has a public +//! constructor, so the supervisor cannot skip a stage or run a workload before +//! [`contract::ReadyBoundary`] exists. +//! +//! [`AgentSpec`] is shared between the workload definition the supervisor +//! submits and the [`contract::SandboxContext`] that `attach` binds to a +//! boundary. + +/// The agent workload to run inside the boundary. +/// +/// Carried by [`contract::SandboxContext`] so a backend's `start_agent` takes no +/// spec; the bound boundary already carries what runs inside it. +#[derive(Debug, Clone)] +pub struct AgentSpec { + /// Entrypoint program. + pub program: String, + /// Entrypoint arguments. + pub args: Vec, + /// Working directory for the entrypoint, if any. + pub workdir: Option, + /// Wall-clock timeout for the entrypoint in seconds (0 = no timeout). + pub timeout_secs: u64, + /// Whether the entrypoint runs interactively (inherits the parent pgrp). + pub interactive: bool, +} + +pub mod contract; diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md new file mode 100644 index 0000000000..07bbc4dbab --- /dev/null +++ b/rfc/0012-isolation-backend/README.md @@ -0,0 +1,394 @@ +--- +authors: + - "@jganoff" +state: review +links: + - https://github.com/NVIDIA/OpenShell/issues/1737 + - https://github.com/NVIDIA/OpenShell/pull/2048 + - https://github.com/NVIDIA/OpenShell/issues/899 + - https://github.com/NVIDIA/OpenShell/issues/981 + - https://github.com/NVIDIA/OpenShell/issues/1511 + - https://github.com/NVIDIA/OpenShell/issues/1650 + - https://github.com/NVIDIA/OpenShell/issues/1680 + - https://github.com/NVIDIA/OpenShell/pull/2606 +--- + +# RFC 0012 - Isolation Backend Interface + +## Summary + +Today the supervisor both builds the workload's isolation boundary and applies its network policy. Because the supervisor runs inside the agent container, the privilege needed to build that boundary sits beside the code it confines. This RFC moves boundary construction and process operations behind a pluggable **Isolation Backend**. The supervisor continues to apply approved network policy through network mediation. + +The compute driver provisions the workload and trusted components. The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. The backend establishes the isolation controls, manages workload processes, and routes egress to network mediation. The same lifecycle supports today's in-pod implementation and future delegated implementations without topology-specific supervisor paths. + +## Motivation + +Boundary construction is embedded in the supervisor, so moving it anywhere else means changing the supervisor. That placement creates three problems: + +- A compromise reaches the boundary-building privilege in the same container. +- Building the boundary inside the agent container requires capabilities that conflict with restricted deployments. Delegating construction removes that requirement but does not guarantee Pod Security Standards compliance. See [codebase-grounding.md](./codebase-grounding.md) and #899 for background. +- Each new placement adds another branch to the supervisor. + +All three come from coupling boundary construction to boundary operation. A common interface lets deployments move privilege without changing the supervisor. + +## Non-goals + +- **Implementing a delegated backend.** Each topology requires its own design and implementation. +- **Changing authorization.** [RFC 0001](../0001-core-architecture/README.md) owns control-plane and sandbox identity. A delegated backend must still authenticate callers and scope them to one boundary. +- **Standardizing backend-internal component coordination.** A backend may coordinate helper, sidecar, or interception processes behind one lifecycle; how those components cooperate is backend-specific, not contract surface. +- **Changing gateway lifecycle or public status.** This RFC adds no gateway activation operation, public phase, or status API, and it does not define how a boundary's effective isolation model is surfaced to operators. + +## Proposal + +The mental model has three roles: + +- The **compute driver** provisions the sandbox instance according to the selected placement of the workload and trusted isolation components. That placement is the **topology**. +- The **Isolation Backend** establishes and operates the topology-specific controls around the workload. It also routes workload egress to network mediation and provides process operations. +- The **logical supervisor** is the trusted control-plane bridge between the gateway and the workload. It drives the backend, handles authorized gateway requests, and applies approved network policy through network mediation. + +Together, network policy, filesystem isolation, syscall filtering, and sandbox identity form the workload's isolation boundary. The roles above enforce that boundary and may run in one process or across several trusted components. Their placement does not change the contract. + +Each active boundary has at most one logical supervisor, which may span multiple coupled processes. The backend routes all workload egress through a per-boundary source, and the supervisor consumes that source. Internal delegation and transport remain topology-private. + +[RFC 0001](../0001-core-architecture/README.md) continues to own sandbox authentication and authorization. In this contract, sandbox identity means binding the authenticated sandbox context to the isolation boundary. + +Admission selects the sandbox's topology and determines its trusted context. The compute driver sets up the topology and gives the logical supervisor a `TopologyDescriptor` describing what it provisioned. The supervisor uses the descriptor to attach the matching Isolation Backend. The backend prepares the required controls before the agent starts. + +```mermaid +flowchart TB + Gateway["Gateway"] -->|"create sandbox"| Driver["Compute driver"] + + subgraph Topology["Driver-provisioned topology (placement varies)"] + Supervisor["Supervisor"] + Backend["Isolation Backend (may coordinate components)"] + subgraph Boundary["Isolation boundary"] + Mediator["Network mediation"] + subgraph Execution["Workload execution environment"] + Workload["Workload"] + end + end + + Supervisor -->|"drives contract"| Backend + Backend -->|"establishes and confirms"| Boundary + Backend -.->|"routes all workload egress to"| Mediator + Supervisor -.->|"applies network policy through"| Mediator + Backend -->|"after Ready: makes admitted agent runnable"| Workload + Workload ==>|"only egress"| Mediator + end + + Driver -->|"resources + TopologyDescriptor"| Supervisor + Mediator -->|"allowed egress"| Egress["Egress"] +``` + +In the in-pod topology, the supervisor drives a backend implemented in the same process. Other topologies may delegate backend operations without changing the supervisor lifecycle. + +A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver owns the sandbox instance and topology lifecycle. + +### Contract invariants + +Six invariants hold for every boundary: + +1. Workload egress is denied except through network mediation for the boundary's lifetime. +2. No untrusted instruction executes until every admitted control applicable to that process is in force. +3. An operation is authorized only when the complete effective policy permits it; network operations are decided through network mediation. There is no silent weakening. +4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the compute driver's provisioned execution environment. +5. Shared infrastructure preserves strict per-boundary lifecycle, policy, identity, enforcement, and cleanup isolation. +6. If the logical supervisor is lost, the boundary remains under its last confirmed enforcement state while supervisor-dependent operations fail closed. Loss of required enforcement ends `Running` and terminates all workload processes within a documented bound; detection and termination may be performed by a trusted node or control-plane actor. Network-mediation unavailability denies outbound connections and never enables direct egress. + +Each backend states its termination bound in its implementation documentation. Loss of the logical supervisor means loss of the components holding the backend lifecycle, not loss of the gateway connection; gateway disconnection follows RFC 0001's reconnection semantics. + +### Provisioning + +Provisioning runs on the control plane, and three rules hold in every topology: + +1. **Admission selects the topology** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. The `TopologyDescriptor` supplied by the compute driver must name that backend, and resolution never falls back to another backend. +2. **The compute driver provisions the topology** and anything the selected backend needs. +3. **The backend establishes standing enforcement before untrusted code runs**, during provisioning or `attach`, depending on the backend. + +If a topology depends on cluster-scoped coverage or registration, admission verifies that the prerequisite covers the boundary's placement before untrusted code runs. + +Every topology provides a trusted cleanup path that does not depend on logical-supervisor availability. + +A compute driver may provision a resource and `TopologyDescriptor` before the control plane assigns it to a sandbox. No untrusted workload runs while the resource is unassigned. After claim or assignment produces a trusted `SandboxContext`, the supervisor calls `attach`; the backend either binds that context to the prepared resource and returns `Bound`, or rejects it as incompatible. Pool creation, claim, reset, release, and recycling remain outside this contract. + +### The topology descriptor + +The driver supplies a descriptor for every topology admitted to this contract, including in-pod and resources prepared before assignment. The common envelope names the backend and carries an opaque payload. + +```rust +struct TopologyDescriptor { + backend_name: String, + version: u32, + payload: Vec, +} +``` + +`version` is the Isolation Backend interface version. Backend name and version match exactly; this contract does not negotiate compatibility ranges. The descriptor is transport-neutral. Provisioning supplies it to the supervisor before `attach`; how it is transported is topology-specific and outside this contract, and every transport preserves one property: workload-controlled input cannot select or modify the descriptor. + +The opaque payload identifies, or gives the backend enough information to resolve, the exact driver-provisioned resource. It may also carry topology-specific endpoint or helper-role information; there are no common topology or role fields. + +Common verification requires: + +- the descriptor's `backend_name` matches the backend required by the admitted topology; +- the descriptor's version is one the supervisor supports, and the resolved backend reports that same version; and +- `SandboxContext` is constructed after the control plane assigns the resource to the admitted sandbox, using authenticated control-plane and trusted supervisor state. + +The supervisor validates the descriptor's common fields and produces a `VerifiedTopologyDescriptor`, then resolves its `backend_name` and version without fallback. Verification does not imply that the opaque payload is valid; the selected backend validates it and atomically binds the provisioned resource to the trusted `SandboxContext` during `attach`. Any failure rejects the sandbox. + +### The lifecycle + +The contract does not prescribe enforcement mechanisms; it standardizes how the supervisor drives whichever backend a deployment admits. + +A backend registers under a `backend_name` and version. The supervisor attaches to the admitted topology and drives the boundary through a fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. + +```text +attach topology + sandbox context -> Bound -> confirm -> Ready -> start_agent -> Running +``` + +```rust +#[async_trait] +trait IsolationBackend: Send + Sync { + fn backend_name(&self) -> &str; + fn version(&self) -> u32; + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError>; +} + +struct SandboxContext { + sandbox_id: SandboxId, + policy: SandboxPolicy, + agent: AgentSpec, +} + +#[async_trait] +trait BoundBoundary: Send { + fn network_mediation_source(&self) -> Arc; + + async fn confirm( + self: Box, + ) -> Result, BackendError>; +} + +#[async_trait] +trait ReadyBoundary: Send { + async fn start_agent( + self: Box, + ) -> Result, BackendError>; +} + +#[async_trait] +trait RunningBoundary: Send + Sync { + fn agent(&self) -> Arc; + fn exec(&self) -> Arc; + fn port_forward(&self) -> Arc; +} +``` + +`AgentSpec` carries the complete admitted agent launch specification, including command, arguments, working directory, timeout, and interactive mode. + +`SandboxContext` carries the admitted create-time policy. [RFC 0002](../0002-agent-driven-policy-management/README.md) defines how network-policy revisions are proposed and approved. Approved revisions reach the supervisor through the existing [`GetSandboxConfig`](../../proto/sandbox.proto) gateway-supervisor contract, described in the [gateway](../../architecture/gateway.md) and [sandbox](../../architecture/sandbox.md#policy-revision-acknowledgement) architecture. The supervisor makes approved network-policy revisions effective through network mediation. If an approved network-policy revision cannot be loaded, it never becomes effective; the configured rejection posture retains the last valid generation or denies network access until a valid generation is loaded. + +The states have normative meanings: + +- **Bound:** the topology descriptor and trusted sandbox context are bound to the same resource, and the network-mediation source is available. No untrusted workload code is running. +- **Ready:** the backend has confirmed standing enforcement for this concrete boundary and is prepared to apply the admitted launch-time controls before untrusted execution. +- **Running:** `start_agent` has made the admitted agent runnable and returned `RunningBoundary`. Every applicable launch-time control was in force before the first untrusted instruction. Whether the backend creates the agent process or releases a held, driver-provisioned execution object is backend-specific; the contract fixes the ordering, not the mechanism. + +`confirm` is the pre-launch commit point. The supervisor calls it only after connecting the boundary's network-mediation source to network mediation. The backend confirms standing enforcement for the concrete boundary and may rely on a trusted provisioning-time or out-of-pod signal tied to that boundary's placement, but not on general placement health alone. + +`attach` rejects a resource already bound to an active boundary. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `attach` or `confirm`, or the supervisor fails network-mediation initialization. + +**Standing enforcement** is established independently of a workload process. **Launch-time controls** must be in force before a process executes its first untrusted instruction. Both `start_agent` and `BoundaryExec::exec` enforce this ordering and preserve the provisioned execution environment. + +`start_agent` is the sole operation that may make the admitted agent runnable. The backend may create or release the process, but workload-controlled code cannot run before `start_agent` applies the required controls. + +`RunningBoundary::agent()` returns a handle for the admitted agent process. Processes started through `BoundaryExec` run in the same boundary and have their own process handles. Every workload process remains within the provisioned execution environment. Any exit of the admitted agent ends `Running`; the backend then terminates every remaining workload process within that environment and rejects further runtime operations, except `wait` as defined below. + +### Runtime operations + +```rust +#[async_trait] +trait BoundaryProcess: Send + Sync { // the agent, or a process started via exec + async fn wait(&self) -> Result; // one stable result where process-exit observation is retained + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError>; + async fn terminate(&self) -> Result<(), BackendError>; // this process and its owned process group +} + +#[async_trait] +trait BoundaryExec: Send + Sync { + async fn exec(&self, spec: ExecSpec) -> Result; +} + +struct ExecSession { // owned; outlives the exec call + process: Arc, + stdin: Option, + stdout: BoundaryOutput, // distinct from stderr for non-PTY exec + stderr: Option, + terminal: Option>, // present when a PTY was requested +} + +#[async_trait] +trait BoundaryPortForward: Send + Sync { + async fn connect(&self, target: LoopbackTarget) -> Result; +} + +#[async_trait] +trait BoundaryTerminal: Send + Sync { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError>; +} +``` + +`ExecSpec` carries command, arguments, environment, working directory, and PTY settings. Streams are owned, non-PTY stdout and stderr remain separate, and PTYs support resize. Port forwarding accepts only validated loopback targets. Exit status and signals are explicit and placement-neutral; a local PID is never the process handle. These operations carry the existing agent, SSH, exec, and forwarding paths behind the contract, and all of them are mandatory conformance. + +`BoundaryProcess::wait` returns one stable exit status or `Terminated` error while the backend retains process-exit observation. + +### Network mediation + +```rust +#[async_trait] +trait NetworkMediationSource: Send + Sync { + async fn accept(&self) -> Result; +} + +struct MediatedConnection { + stream: BoundaryDuplexStream, + binary_identity: Result, +} +``` + +`NetworkMediationSource` supplies outbound connections from one boundary to supervisor-owned network mediation. The backend routes all workload egress through that source and authoritatively associates each connection with the boundary without relying solely on workload-provided data. Capture, transport, placement, and coordination are backend-private. + +Every topology may use the same supervisor-owned mediation libraries or services; the source does not require a backend-specific policy engine. + +Shared implementations isolate each boundary's state and enforcement. Failure or teardown of one boundary cannot weaken another. Network-mediation unavailability never enables direct egress. + +### Binary identity + +The backend resolves executable identity for every accepted connection and delivers the result on `MediatedConnection` before network mediation evaluates policy. + +```rust +struct BinaryIdentity { + binary_path: PathBuf, // absolute executable path + binary_digest: Option, // bytes of the resolved executable object + ancestors: Vec, // nearest first + cmdline_paths: Vec, // diagnostic context; never authorizes +} +``` + +Identity describes the executable identity resolved for the accepted connection before policy evaluation. Paths are expressed in the workload's filesystem namespace. + +If binary identity cannot be resolved, the connection is denied. `ResolveError` reports that failure. A missing digest is represented as `None`. How a backend resolves identity is implementation-specific. + +Every identity field used for authorization is obtained by a trusted component from boundary or kernel state, rather than accepted as a workload claim. The result is bound to the active boundary and accepted connection; a transport tuple or workload-supplied identifier alone is not authoritative. Workload-supplied identity may be retained only as non-authorizing diagnostic context. If attribution is ambiguous or any required identity field cannot be established, the connection is denied. + +Binary identity is mandatory conformance: RFC 0002 makes it part of the outbound-policy baseline. There is no capability flag and no mode that exempts a backend from resolving identity. + +### The supervisor sequence + +The logical supervisor resolves `backend_name` and version through a trusted implementation registry. Adding a backend adds an implementation and registration, not branches in lifecycle, proxy, SSH, or session code. Delegated transport and coordination remain backend-private. + +The supervisor runs the same sequence for every backend: + +1. Obtain the `TopologyDescriptor` and trusted `SandboxContext`. +2. Verify the descriptor and resolve its `backend_name` and version without fallback. +3. Call `attach` to obtain `Bound`. +4. Connect the boundary's `NetworkMediationSource` to network mediation. +5. Call `confirm` to obtain `Ready`, then `start_agent` to obtain `Running`. +6. Use the returned runtime handles for agent wait, `exec`, and port forwarding while network mediation consumes outbound connections. + +This RFC supersedes RFC 0001's fixed in-sandbox supervisor placement and its assignment of topology-specific isolation controls to that process, generalizing the supervisor into a logical role. RFC 0001's authentication, sandbox-identity, outbound-connection, session, and reconnection requirements continue to apply. The component hosting the logical supervisor holds the required outbound gateway connection. A driver-hosted or shared supervisor routes gateway `exec`, SSH, and forwarding requests through the backend; the gateway does not initiate a connection to the boundary. + +### Failure semantics + +Every failure carries a machine-readable kind for supervisor status mapping: + +```rust +enum BackendErrorKind { Invalid, Denied, Unavailable, Failed, Terminated } +``` + +`Invalid` covers descriptor, version, and backend mismatches; `Denied` covers authenticated attachment rejection; `Unavailable` covers transient inability to serve an operation; `Failed` covers other backend faults; and `Terminated` reports boundary or workload termination, or an operation against an inactive boundary. An error never advances the lifecycle or authorizes an operation, and backend selection never falls back. + +A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per provisioned topology. If it does not return `Bound`, the topology is reclaimed rather than reused. + +Failures resolve as follows: + +- an `attach` or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the driver to reclaim the topology; +- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the driver reclaims the topology; +- if `exec` or port-forward `connect` fails, the backend terminates any process or closes any connection created by that attempt while the boundary otherwise remains active; +- after `Running`, supervisor or enforcement loss follows invariant 6; when enforcement loss ends the agent, `BoundaryProcess::wait` fails with `BackendErrorKind::Terminated` where process-exit observation survives; +- network-mediation errors yield no authorized connection and do not by themselves end `Running`; and +- retained runtime handles and the network-mediation source reject new operations whenever the boundary ends, except `BoundaryProcess::wait` where the backend can still return its stable result. + +Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the compute driver reclaims or deprovisions the topology. If normal backend cleanup is unavailable, the compute driver uses the topology's trusted cleanup path to terminate the execution environment and invalidate the binding before reclaim or reuse. On normal agent exit, `BoundaryProcess::wait` returns the stable exit status. A retained `wait` result may outlive teardown. + +### Topologies + +The contract fixes the roles; a topology fixes their placement. Components may be co-located with the workload or hosted in trusted services, and one component may implement multiple roles. Every arrangement admitted to this contract preserves the same lifecycle, interfaces, and invariants. Actual containment depends on the workload's kernel relationship to the trusted components. The non-normative [topology matrix](./topology-matrix.md) catalogs representative placements. + +## Implementation plan + +This RFC defines the contract; implementation lands in three phases: + +1. **Contract.** Add the common types, descriptor handling, registry, and explicit backend selection from deployment configuration. +2. **Co-located backend.** Implement the co-located backend behind a deployment flag and route agent launch, egress interception, the network-mediation source, SSH, `exec`, and forwarding through it without changing behavior. +3. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus descriptor verification, lifecycle ordering, runtime operations, and failure semantics. Make the co-located backend the default after parity validation. Parity covers the agent, binary identity, SSH, `exec`, and forwarding paths; enablement also closes the in-pod egress gaps pinned in [codebase-grounding.md](./codebase-grounding.md), which parity alone would preserve. + +Existing placements remain outside this contract until their backend is implemented and admitted; they do not claim conformance. Delegated backends remain separate design and implementation work. + +## Risks + +| Risk | Mitigation | +|---|---| +| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep the responsibility boundary explicit: the compute driver owns, provisions, and deprovisions the topology; the backend binds and operates the active boundary. The same component may implement both roles. | +| Contract conformance could be mistaken for equivalent isolation across topologies. | Treat conformance as behavioral, not as a security-strength rating. Document and validate each topology's actual containment and reject policy it cannot enforce. | +| Shared backend or network-mediation components concentrate privilege and failure impact. | Isolate state, connection attribution, enforcement, and control authority per boundary. Failure of one boundary must not weaken another or enable direct egress. | +| The mandatory contract may exclude otherwise useful but incomplete backends. | Keep the network-mediation source, binary identity, process control, `exec`, and port forwarding mandatory. An incomplete backend does not claim conformance or silently degrade. | +| A future topology may not fit the lifecycle or interfaces. | Keep placement and coordination backend-private. Add versioned contract surface only when a concrete implementation requires new common semantics. | +| A component restart may interrupt boundary operation. | Keep the last confirmed enforcement state in force and deny supervisor-dependent operations. | + +## Alternatives + +### Keep isolation embedded in the supervisor + +OpenShell could keep the current in-pod design and add topology-specific supervisor and compute-driver paths as new requirements arise. + +Doing nothing avoids a new interface, but retains privileged boundary construction beside the workload. Implementing each delegated topology as a one-off supervisor change moves that privilege for one placement but accretes topology-specific supervisor behavior. The proposed contract instead keeps one supervisor lifecycle while allowing the topology to change. + +### Extend the compute-driver contract + +The compute driver could own both provisioning and active-boundary operation. + +This is natural for topologies such as MXC, and the same component may implement both responsibilities. The interfaces remain distinct because they serve different callers and lifecycles: the gateway uses the compute driver to provision and deprovision resources, while the supervisor uses the Isolation Backend to operate an active boundary. Combining them would couple runtime policy, identity, network mediation, and process operations to the gateway-facing driver API. + +### Start with a remote backend service + +The contract could be expressed as a gRPC service or plugin ABI rather than an in-process Rust contract. [RFC 0001](../0001-core-architecture/README.md) chose gRPC for its gateway-facing drivers, so the question applies here. + +The callers differ. A gateway driver is a control-plane peer with its own release cycle, while the Isolation Backend is driven by the supervisor that operates the boundary, and the co-located topology needs no transport at all. Starting in-process serves that case directly and lets delegated implementations carry their own transport behind the same interface. A transport-bearing surface is not precluded: it is versioned contract surface, added when a concrete delegated backend requires it. + +### Standardize topology and capabilities + +The contract could expose common topology roles, placement fields, capability flags, and recovery behavior so the supervisor can compose backend components. + +That would make known deployments explicit, but it would also encode current topology assumptions and introduce capability-dependent supervisor paths. The proposal keeps placement and coordination in the opaque descriptor, requires one baseline contract, and uses the non-normative topology matrix to document representative arrangements. + +## Prior art + +- **Driver-backed subsystems (CRI/CNI/CSI).** Kubernetes factors runtime, networking, and storage into pluggable driver contracts so the orchestrator drives one interface while implementations vary. RFC 0001 describes OpenShell's other subsystems the same way; this RFC specifies the one it left open: isolation. +- **Istio privilege placement.** Init-sidecar and node-agent modes demonstrate that network setup can move without changing the policy data path. OpenShell keeps its identity-aware proxy. +- **CRI exec/attach/port-forward.** `exec` and `connect` follow CRI's `Exec` and `PortForward` shape; lifecycle and network mediation remain OpenShell-specific. + +## Open questions + +None. + +## Appendix: codebase grounding + +The claims this RFC makes about the current system, and the current-system +context behind its design, are verified with file:line references in the +supporting file [codebase-grounding.md](./codebase-grounding.md) +(against upstream commit `905b554c`, after proxy egress pipeline consolidation). diff --git a/rfc/0012-isolation-backend/codebase-grounding.md b/rfc/0012-isolation-backend/codebase-grounding.md new file mode 100644 index 0000000000..4767dc6d58 --- /dev/null +++ b/rfc/0012-isolation-backend/codebase-grounding.md @@ -0,0 +1,26 @@ +# Codebase grounding (supporting material for RFC 0012) + +This non-normative file grounds RFC 0012's claims about the current system. + +References are pinned to `905b554c` (proxy egress pipeline consolidation, #2373). Permalinks use +`https://github.com/NVIDIA/OpenShell/blob/905b554c/#L`; the `rg` +patterns locate the same code on newer revisions. + +| Claim | Reference | +|---|---| +| Combined-topology agent container's seven capabilities | `crates/openshell-driver-kubernetes/src/driver.rs:2538` (base `SYS_ADMIN`/`NET_ADMIN`/`SYS_PTRACE`/`SYSLOG`), `:2544` (`SETUID`/`SETGID`/`DAC_READ_SEARCH` under userns). `rg -n -e SYS_ADMIN -e NET_ADMIN -e SYS_PTRACE -e SYSLOG -e SETUID -e SETGID -e DAC_READ_SEARCH crates/openshell-driver-kubernetes/src/driver.rs` | +| Spec already separated from the netns handle | `crates/openshell-supervisor-process/src/process.rs:527`/`:535` (`ProcessHandle::spawn` takes `netns: Option<&NetworkNamespace>`) | +| Six `setns(CLONE_NEWNET)` call sites the contract's runtime interfaces replace (agent launch, SSH exec and forward, supervisor sessions, and namespace construction or entry); plus `CLONE_NEWNS` at `:449` (`unshare`, private mount namespace) and `:480` (`setns`, enter mount namespace, added for sidecar topology) | `process.rs:695`, `ssh.rs:653`/`:1262`, `supervisor_session.rs:735`, `netns/mod.rs:226`/`:342` (`rg -n "CLONE_NEWNET" crates/openshell-supervisor-process`) | +| `nft`-absent fail-open (the invariant bug), in-pod path only; the sidecar path uses `nft` with an `iptables-legacy` fallback and returns an error if neither establishes enforcement | `crates/openshell-supervisor-process/src/netns/mod.rs:265`; logs and returns `Ok(())` at `:277`; sidecar fallback at `:459`-`:471` | +| In-pod nftables ceiling is accept-by-default and rejects only TCP and UDP, so reading it back does not prove "only the proxy can egress" | `crates/openshell-supervisor-process/src/netns/nft_ruleset.rs:53` (`type filter hook output priority 0; policy accept`), `:56`-`:92` (proxy/loopback/established accept, then `reject` for IPv4 and IPv6 TCP and UDP only at `:106`+; other protocols and raw sockets pass once the host forwards the subnet). `rg -n "policy accept" crates/openshell-supervisor-process/src/netns` | +| Compute driver owns the execution domain (cgroup/resources, security context, device allocation set on the pod by the driver, not the supervisor) | `crates/openshell-driver-kubernetes/src/driver.rs` builds the pod/container spec; `rg -n -e securityContext -e resources -e 'cdi\.k8s\.io' -e devices crates/openshell-driver-kubernetes/src` | +| VM driver enables forwarding/MASQUERADE (host-forward assumption is load-bearing) | `crates/openshell-driver-vm/src/runtime.rs:418`/`:437` | +| No `StartSandbox` RPC (create and start fused; no driver start gate) | `proto/compute_driver.proto` has `CreateSandbox`/`StopSandbox`/`DeleteSandbox` only | +| Gateway already speaks exec/session/port-forward; no lifecycle `Attach` (`AttachSandboxProvider` exists but attaches a provider record to a running sandbox, not the isolation lifecycle) | `proto/openshell.proto` (`ExecSandbox`, `ExecSandboxInteractive`, `CreateSshSession`, `ForwardTcp`, `AttachSandboxProvider`) | +| Agent command via CLI/`SANDBOX_COMMAND`; no admission-bound spec field today; the `sleep infinity` placeholder resolves `sleep` from the agent image's own filesystem | `crates/openshell-sandbox/src/main.rs:601`; K8s driver sets `sleep infinity` via `SANDBOX_COMMAND` at `driver.rs:2937` (`rg -n "sleep infinity" crates/openshell-driver-kubernetes/src`) | +| Init containers: `copy-self` (trusted, the OpenShell binary) and `workspace-init` (runs as root from the agent's own image, so its executables are image-provided); sidecar topology adds `openshell-network-init` (nftables setup, `NET_ADMIN`/`NET_RAW`/`CHOWN`/`FOWNER`) and `openshell-supervisor-network` runtime sidecar | `driver.rs:423` (`WORKSPACE_INIT_CONTAINER_NAME`), `:1506` (`copy-self` invocation), `:2113` (workspace-init container), `:1423` (`SUPERVISOR_NETWORK_INIT_CONTAINER_NAME`), `:1426` (`SUPERVISOR_NETWORK_SIDECAR_NAME`); `rg -n -e restart_policy -e workspace-init -e openshell-network-init -e openshell-supervisor-network crates/openshell-driver-kubernetes/src` | +| Network policy is OPA per-CONNECT, not the boundary; identity via procfs | `crates/openshell-supervisor-network/src/opa.rs` (`NetworkInput`: `binary_path`/`binary_sha256`/`ancestors`/`cmdline_paths`), `procfs.rs`, glued in `proxy.rs:1955` (`authorize_egress_intent`; `NetworkInput` built at `:2032`) | +| Network enforcement already shares one implementation across placements: the combined path constructs and retains networking before agent launch, while the network-only sidecar owns the proxy, policy polling, and a topology-private control channel | `crates/openshell-sandbox/src/lib.rs:355` (`networking`), `:389` (`sidecar_control_server`), `:590` (`run_policy_poll_loop`), `:686` (`run_process`, after networking setup), `:736` (network-only sidecar lifecycle). `rg -n -e 'let networking' -e sidecar_control_server -e run_policy_poll_loop -e 'Network-only sidecar mode' crates/openshell-sandbox/src/lib.rs` | +| Binary identity is resolved after the connection is accepted (`/proc/net/tcp` inode lookup, socket-owner search, `/proc//exe`, then PPID walking), so executable identity and ancestry describe state observed through trusted kernel interfaces during policy evaluation rather than an atomic snapshot at `connect()` | `crates/openshell-supervisor-network/src/procfs.rs:165` (`resolve_tcp_peer_binary`), `:343` (`parse_proc_net_tcp`), `:441` (`find_socket_inode_owners`), `:227` (`read_ppid`), `:243` (`collect_ancestor_binaries`). `rg -n -e resolve_tcp_peer_binary -e collect_ancestor_binaries crates/openshell-supervisor-network/src` | +| Identity display paths come from reading `/proc//exe`, and the digest covers that live executable object at resolution time rather than a re-read of the display path | `crates/openshell-supervisor-network/src/procfs.rs:127` (`binary_path`), `:134`-`:135` (`read_link` of `/proc//exe`), `proxy.rs:1800`/`:1816` (binary and ancestors both via `verify_or_cache_process_exe`), `identity.rs:107` (hashes `/proc//exe`); digest rationale at `procfs.rs:117`-`:122`. `rg -n -e 'fn binary_path' -e verify_or_cache_process_exe crates/openshell-supervisor-network/src` | +| Static privilege ceiling on every spawned process; OPA never evaluates exec | `process.rs:527` (`ProcessHandle::spawn`), `:710`/`:812` (`drop_privileges` call sites), `:721`/`:818`-`:819` (sandbox enforcement); SSH reaches the same `enter_netns_and_sandbox` path (`ssh.rs:1245`) | diff --git a/rfc/0012-isolation-backend/topology-matrix.md b/rfc/0012-isolation-backend/topology-matrix.md new file mode 100644 index 0000000000..8dc14d84ba --- /dev/null +++ b/rfc/0012-isolation-backend/topology-matrix.md @@ -0,0 +1,36 @@ +# Topology matrix + +This non-normative matrix compares representative mappings of RFC 0012's logical +roles. It records role placement, sharing, and relationship to the workload +kernel; it does not select a deployment or establish conformance. + +## Representative placements + +| Pattern | Logical supervisor and network-mediation placement | Backend placement | Workload-kernel relationship | Topology status | +|---|---|---|---|---| +| **Co-located/in-pod** | With the workload | In the supervisor process | Trusted components share the workload's host, guest, or application kernel, depending on the runtime | Placement implemented (original topology) | +| **Same-pod composite** | Spans the workload-local supervisor process and, when used, a network-mediation sidecar | In the workload-local supervisor process | Components share the workload's kernel | Placement implemented (#2076) | +| **Delegated backend components** | With the workload and any delegated mediation component | A node or remote helper establishes some controls behind a workload-local backend | Depends on which trusted components remain with the workload | Placement proposed (#2606) | +| **Driver-hosted/shared service** | With the compute driver or another trusted service; no in-sandbox supervisor process is required | May be co-located with the logical supervisor; one host may operate many isolated boundaries | Depends on the workload runtime | Placement proposed | + +## Durable rules + +- Every active boundary has one verified descriptor, one trusted + `SandboxContext`, and at most one logical supervisor, which may span multiple + coupled processes. +- Physical processes and listeners may be shared, but lifecycle state, policy, + binary identity, enforcement, and cleanup remain isolated per boundary. +- Moving a privileged component does not itself provide kernel separation. + +## Kernel relationships + +| Relationship | Meaning | +|---|---| +| **Shared host kernel** | The workload and the trusted components relied on for containment run on the host's kernel. | +| **Shared guest or application kernel** | The workload and those trusted components share one isolated kernel: a VM guest kernel or a userspace application kernel. | +| **Kernel-separated** | The trusted components relied on for containment run outside the workload's kernel. | + +## Status + +This matrix is non-normative. It illustrates implementations of RFC 0012; it +does not extend the contract. From 5b589abd5afadd6cd56b983c0af34f980882cf83 Mon Sep 17 00:00:00 2001 From: Drew Newberry <385+drew@users.noreply.github.com> Date: Wed, 26 Aug 2026 04:53:20 +0000 Subject: [PATCH 2/3] refactor(supervisor): add backend-neutral boundary primitives Signed-off-by: Drew Newberry <385+drew@users.noreply.github.com> --- Cargo.lock | 4 + .../openshell-supervisor-network/Cargo.toml | 3 + .../src/identity_source.rs | 168 +++++ .../src/l7/tls.rs | 10 +- .../openshell-supervisor-network/src/lib.rs | 1 + .../openshell-supervisor-process/Cargo.toml | 2 + .../src/boundary_exec.rs | 695 ++++++++++++++++++ .../src/boundary_io.rs | 317 ++++++++ .../openshell-supervisor-process/src/lib.rs | 2 + .../src/managed_children.rs | 154 +++- .../src/netns/nft_ruleset.rs | 71 +- .../src/sandbox/linux/seccomp.rs | 39 + .../openshell-supervisor-process/src/ssh.rs | 4 +- 13 files changed, 1427 insertions(+), 43 deletions(-) create mode 100644 crates/openshell-supervisor-network/src/identity_source.rs create mode 100644 crates/openshell-supervisor-process/src/boundary_exec.rs create mode 100644 crates/openshell-supervisor-process/src/boundary_io.rs diff --git a/Cargo.lock b/Cargo.lock index 549b1fc834..b9d4b7a32f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4449,6 +4449,7 @@ name = "openshell-supervisor-network" version = "0.0.0" dependencies = [ "apollo-parser", + "async-trait", "aws-credential-types", "aws-sigv4", "aws-smithy-runtime-api", @@ -4464,6 +4465,7 @@ dependencies = [ "libc", "miette", "openshell-core", + "openshell-isolation", "openshell-ocsf", "openshell-policy", "openshell-router", @@ -4502,6 +4504,7 @@ name = "openshell-supervisor-process" version = "0.0.0" dependencies = [ "anyhow", + "async-trait", "base64 0.22.1", "bytes", "capctl", @@ -4512,6 +4515,7 @@ dependencies = [ "miette", "nix 0.29.0", "openshell-core", + "openshell-isolation", "openshell-ocsf", "openshell-policy", "rand 0.10.2", diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 34d9c32a47..5ae4bf1f94 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -12,11 +12,14 @@ rust-version.workspace = true [dependencies] openshell-core = { path = "../openshell-core", features = ["oauth"] } +openshell-isolation = { path = "../openshell-isolation" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-router = { path = "../openshell-router" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } +async-trait = "0.1" + apollo-parser = { workspace = true } aws-sigv4 = { version = "1", features = ["sign-http", "http1"] } aws-credential-types = { version = "1", features = ["hardcoded-credentials"] } diff --git a/crates/openshell-supervisor-network/src/identity_source.rs b/crates/openshell-supervisor-network/src/identity_source.rs new file mode 100644 index 0000000000..0e5960aa8e --- /dev/null +++ b/crates/openshell-supervisor-network/src/identity_source.rs @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The in-pod binary-identity resolver (RFC 0012 runtime contract). +//! +//! RFC 0012 delivers executable identity on every +//! [`MediatedConnection`](openshell_isolation::contract::MediatedConnection): +//! the backend resolves identity for the accepted connection before mediation. +//! An unresolved identity denies that connection. This is the in-pod +//! resolution mechanism — procfs, keyed by the workload-side TCP peer port — +//! kept in this crate on purpose: the proxy that consumes identity is here, and +//! so are procfs and the binary identity cache. Stronger backends may use a +//! different resolution mechanism without changing the contract. The result +//! type lives in the lower `openshell-isolation` crate (network -> isolation -> +//! core, acyclic). +//! +//! The legacy listener still resolves identity in the proxy hot path. The RFC +//! 0012 co-located source invokes this resolver before returning each accepted +//! connection, so mediation consumes the bound identity result. + +use std::sync::Arc; +use std::sync::atomic::AtomicU32; + +use openshell_isolation::contract::{BinaryIdentity, ResolveError, Sha256Digest}; + +/// In-pod binary-identity resolver: reads and hashes the executable resolved +/// for an accepted connection from procfs. Resolution fails closed; it never +/// fabricates identity fields. +#[derive(Clone)] +pub struct ProcfsIdentityResolver { + /// The workload entrypoint PID, whose network namespace owns the peer + /// sockets the proxy resolves. Published once the agent starts. + pub entrypoint_pid: Arc, +} + +impl ProcfsIdentityResolver { + /// Resolve the executable identity behind an accepted workload connection. + pub fn resolve_connection( + &self, + workload_addr: std::net::SocketAddr, + proxy_addr: std::net::SocketAddr, + ) -> Result { + // procfs resolution is Linux-only; on other targets the supervisor has + // no procfs to read, so resolution fails closed. + #[cfg(target_os = "linux")] + { + self.resolve_via_procfs(workload_addr, proxy_addr) + } + #[cfg(not(target_os = "linux"))] + { + let _ = (workload_addr, proxy_addr); + Err(ResolveError::Failed( + "no procfs on this platform; identity resolution unavailable".to_string(), + )) + } + } +} + +#[cfg(target_os = "linux")] +impl ProcfsIdentityResolver { + fn resolve_via_procfs( + &self, + workload_addr: std::net::SocketAddr, + proxy_addr: std::net::SocketAddr, + ) -> Result { + use std::sync::atomic::Ordering; + + let entrypoint_pid = self.entrypoint_pid.load(Ordering::Acquire); + if entrypoint_pid == 0 { + // No workload yet: nothing to attribute the connection to. Fail + // closed so a binary-scoped rule cannot match an unattributed peer. + return Err(ResolveError::NotFound); + } + + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let owners = crate::procfs::resolve_tcp_peer_socket_owners(entrypoint_pid, connection) + .map_err(|_| ResolveError::NotFound)?; + let mut identities = Vec::with_capacity(owners.owners.len()); + for owner in owners.owners { + identities.push(Self::resolve_owner(owner.pid, entrypoint_pid)?); + } + let Some(identity) = identities.first().cloned() else { + return Err(ResolveError::NotFound); + }; + if identities.iter().skip(1).any(|candidate| { + candidate.binary_path != identity.binary_path + || candidate.binary_digest != identity.binary_digest + || candidate.ancestors != identity.ancestors + || candidate.cmdline_paths != identity.cmdline_paths + }) { + return Err(ResolveError::Failed( + "shared socket owners have different policy identities".to_string(), + )); + } + Ok(identity) + } + + fn resolve_owner(owner_pid: u32, entrypoint_pid: u32) -> Result { + let binary_path = crate::procfs::binary_path(owner_pid.cast_signed()) + .map_err(|error| ResolveError::Failed(error.to_string()))?; + + // Hash the live `/proc//exe` object, not the reopened resolved + // path: opening the magic symlink pins the inode the process is actually + // executing, so a post-resolution swap of the path cannot launder the + // hash. A missing digest is `None`, never an empty string, and an + // unhashable binary fails closed rather than asserting an identity the + // resolver could not verify. + let exe = std::path::PathBuf::from(format!("/proc/{owner_pid}/exe")); + let binary_digest = match crate::procfs::file_sha256(&exe) { + Ok(digest) => Some(digest.parse::()?), + Err(_) => { + return Err(ResolveError::Failed( + "could not hash resolved executable; refusing to assert identity".to_string(), + )); + } + }; + + let ancestors = crate::procfs::collect_ancestor_binaries(owner_pid, entrypoint_pid); + let mut exclude = ancestors.clone(); + exclude.push(binary_path.clone()); + let cmdline_paths = + crate::procfs::collect_cmdline_paths(owner_pid, entrypoint_pid, &exclude); + + Ok(BinaryIdentity { + binary_path, + binary_digest, + ancestors, + cmdline_paths, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Stands in for the mediation service: a binary-scoped rule can only be + /// authorized by a resolved identity carrying the fields it requires. + fn admits_binary_rule(result: Result) -> bool { + matches!(result, Ok(identity) if identity.binary_digest.is_some()) + } + + #[test] + fn fails_closed_before_the_workload_starts() { + // entrypoint_pid == 0 means no agent yet; identity must fail closed so a + // binary-scoped rule cannot be satisfied by an unattributed connection. + let resolver = ProcfsIdentityResolver { + entrypoint_pid: Arc::new(AtomicU32::new(0)), + }; + assert!(!admits_binary_rule(resolver.resolve_connection( + "127.0.0.1:12345".parse().unwrap(), + "127.0.0.1:3128".parse().unwrap(), + ))); + } + + #[test] + fn unknown_peer_fails_closed() { + // A peer port no live workload connection owns must resolve to an error, + // never a fabricated identity. + let resolver = ProcfsIdentityResolver { + entrypoint_pid: Arc::new(AtomicU32::new(u32::MAX - 1)), + }; + assert!(!admits_binary_rule(resolver.resolve_connection( + "127.0.0.1:1".parse().unwrap(), + "127.0.0.1:3128".parse().unwrap(), + ))); + } +} diff --git a/crates/openshell-supervisor-network/src/l7/tls.rs b/crates/openshell-supervisor-network/src/l7/tls.rs index 2275a60d34..d3def44743 100644 --- a/crates/openshell-supervisor-network/src/l7/tls.rs +++ b/crates/openshell-supervisor-network/src/l7/tls.rs @@ -17,7 +17,6 @@ use std::io::BufReader; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use tokio::io::{AsyncRead, AsyncWrite}; -use tokio::net::TcpStream; use tokio_rustls::{TlsAcceptor, TlsConnector}; const MAX_CACHED_CERTS: usize = 256; @@ -170,11 +169,14 @@ impl ProxyTlsState { /// Accept TLS from a sandbox client, presenting a dynamic cert for the hostname. /// /// Returns a TLS stream that can be used for plaintext HTTP inspection. -pub async fn tls_terminate_client( - client: TcpStream, +pub async fn tls_terminate_client( + client: S, tls_state: &ProxyTlsState, hostname: &str, -) -> Result { +) -> Result +where + S: AsyncRead + AsyncWrite + Unpin + Send, +{ let acceptor = tls_state.acceptor_for(hostname)?; let tls_stream = acceptor.accept(client).await.into_diagnostic()?; Ok(tls_stream) diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index 4fec48b300..a828f75fba 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -9,6 +9,7 @@ //! aggregate them. pub mod identity; +pub mod identity_source; pub mod inference_routes; pub mod l7; pub mod opa; diff --git a/crates/openshell-supervisor-process/Cargo.toml b/crates/openshell-supervisor-process/Cargo.toml index 2e2120f1d0..5a2ba05f64 100644 --- a/crates/openshell-supervisor-process/Cargo.toml +++ b/crates/openshell-supervisor-process/Cargo.toml @@ -12,10 +12,12 @@ rust-version.workspace = true [dependencies] openshell-core = { path = "../openshell-core" } +openshell-isolation = { path = "../openshell-isolation" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } anyhow = { workspace = true } +async-trait = "0.1" base64 = { workspace = true } bytes = { workspace = true } hex = "0.4" diff --git a/crates/openshell-supervisor-process/src/boundary_exec.rs b/crates/openshell-supervisor-process/src/boundary_exec.rs new file mode 100644 index 0000000000..00d5239f29 --- /dev/null +++ b/crates/openshell-supervisor-process/src/boundary_exec.rs @@ -0,0 +1,695 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Co-located implementation of RFC 0012 in-boundary exec. + +use std::collections::HashMap; +use std::os::fd::{AsRawFd, OwnedFd}; +use std::process::{Child, Command, Stdio}; +use std::sync::Arc; + +use async_trait::async_trait; +use nix::pty::{Winsize, openpty}; +use nix::sys::signal::{Signal, killpg}; +use nix::unistd::Pid; +use openshell_core::policy::SandboxPolicy; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation::contract::{ + BackendError, BoundaryExec, BoundaryExitStatus, BoundaryInput, BoundaryOutput, BoundaryProcess, + BoundarySignal, BoundaryTerminal, ExecSession, ExecSpec, +}; + +use crate::process::{ProcessEnforcementMode, ResolvedProcessIdentity}; + +/// The co-located executor. Every spawn reuses the same admitted policy and +/// execution-environment controls while taking a fresh provider credential +/// snapshot. +#[derive(Clone)] +pub struct LocalBoundaryExec { + policy: SandboxPolicy, + base_workdir: Option, + netns_fd: Option>, + proxy_url: Option, + ca_file_paths: Option>, + provider_credentials: ProviderCredentialState, + user_environment: HashMap, + resolved_identity: ResolvedProcessIdentity, + enforcement_mode: ProcessEnforcementMode, + runtime: Arc, +} + +impl LocalBoundaryExec { + /// Construct one executor for an active co-located boundary. + #[allow(clippy::too_many_arguments)] + #[must_use] + pub fn new( + policy: SandboxPolicy, + base_workdir: Option, + netns_fd: Option>, + proxy_url: Option, + ca_file_paths: Option>, + provider_credentials: ProviderCredentialState, + user_environment: HashMap, + resolved_identity: ResolvedProcessIdentity, + enforcement_mode: ProcessEnforcementMode, + runtime: Arc, + ) -> Self { + Self { + policy, + base_workdir, + netns_fd, + proxy_url, + ca_file_paths, + provider_credentials, + user_environment, + resolved_identity, + enforcement_mode, + runtime, + } + } + + fn command(&self, spec: &ExecSpec) -> Result { + if spec.program.is_empty() { + return Err(BackendError::Process("exec program is empty".to_string())); + } + let mut command = Command::new(&spec.program); + command.args(&spec.args); + let effective_workdir = spec.workdir.as_deref().or(self.base_workdir.as_deref()); + let (session_user, session_home) = + crate::process::session_user_and_home(&self.policy, effective_workdir); + crate::ssh::apply_child_env( + &mut command, + &session_home, + &session_user, + if spec.pty { "xterm-256color" } else { "dumb" }, + self.proxy_url.as_deref(), + self.ca_file_paths.as_deref(), + &self.provider_credentials.child_env_with_gcp_resolved(), + &self.user_environment, + ); + for (key, value) in &spec.env { + if !key.starts_with("OPENSHELL_") { + command.env(key, value); + } + } + if let Some(workdir) = spec.workdir.as_deref().or(self.base_workdir.as_deref()) { + command.current_dir(workdir); + } + Ok(command) + } + + fn prepare_sandbox( + &self, + workdir: Option<&str>, + ) -> Result, BackendError> { + #[cfg(target_os = "linux")] + { + if self.enforcement_mode.enforces_child_sandbox() { + crate::sandbox::linux::log_sandbox_readiness(&self.policy, workdir); + } + crate::process::prepare_child_sandbox(&self.policy, workdir, self.enforcement_mode) + .map_err(|error| BackendError::Process(error.to_string())) + } + #[cfg(not(target_os = "linux"))] + { + let _ = workdir; + Ok(None) + } + } + + fn spawn_piped(&self, spec: &ExecSpec) -> Result { + self.runtime.ensure_active()?; + let mut command = self.command(spec)?; + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let effective_workdir = spec.workdir.as_deref().or(self.base_workdir.as_deref()); + #[cfg(target_os = "linux")] + let prepared = self.prepare_sandbox(effective_workdir)?; + crate::ssh::unsafe_pty::install_pre_exec_no_pty( + &mut command, + self.policy.clone(), + effective_workdir.map(str::to_string), + self.netns_fd.as_deref().map(AsRawFd::as_raw_fd), + self.resolved_identity, + self.enforcement_mode, + #[cfg(target_os = "linux")] + prepared, + ); + #[cfg(target_os = "linux")] + let mut child_registry = crate::managed_children::lock(); + let mut child = command + .spawn() + .map_err(|error| BackendError::Process(error.to_string()))?; + let pid = child.id(); + let process_terminal = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let signal_lock = Arc::new(std::sync::Mutex::new(())); + if let Err(error) = + self.runtime + .register_process_group(pid, process_terminal.clone(), signal_lock.clone()) + { + let _ = killpg( + Pid::from_raw(i32::try_from(pid).unwrap_or(i32::MAX)), + Signal::SIGKILL, + ); + let _ = child.wait(); + return Err(error); + } + #[cfg(target_os = "linux")] + let managed_child = child_registry.register(pid); + #[cfg(target_os = "linux")] + drop(child_registry); + let stdin = child.stdin.take().map(|file| -> BoundaryInput { + let fd: OwnedFd = file.into(); + Box::new(tokio::fs::File::from_std(std::fs::File::from(fd))) + }); + let stdout = child + .stdout + .take() + .map(|file| -> BoundaryOutput { + let fd: OwnedFd = file.into(); + Box::new(tokio::fs::File::from_std(std::fs::File::from(fd))) + }) + .ok_or_else(|| BackendError::Process("exec stdout pipe missing".to_string()))?; + let stderr = child.stderr.take().map(|file| -> BoundaryOutput { + let fd: OwnedFd = file.into(); + Box::new(tokio::fs::File::from_std(std::fs::File::from(fd))) + }); + let process = Arc::new(LocalExecProcess::new( + child, + pid, + self.runtime.clone(), + process_terminal, + signal_lock, + #[cfg(target_os = "linux")] + managed_child, + )); + Ok(SpawnedExec { + session: Some(ExecSession { + process: process.clone(), + stdin, + stdout, + stderr, + terminal: None, + }), + process, + armed: true, + }) + } + + fn spawn_pty(&self, spec: &ExecSpec) -> Result { + self.runtime.ensure_active()?; + let winsize = Winsize { + ws_row: 24, + ws_col: 80, + ws_xpixel: 0, + ws_ypixel: 0, + }; + let pty = openpty(Some(&winsize), None) + .map_err(|error| BackendError::Process(error.to_string()))?; + let master = std::fs::File::from(pty.master); + let slave = std::fs::File::from(pty.slave); + let slave_fd = slave.as_raw_fd(); + let input = master + .try_clone() + .map_err(|error| BackendError::Process(error.to_string()))?; + let output = master + .try_clone() + .map_err(|error| BackendError::Process(error.to_string()))?; + let stdin = slave + .try_clone() + .map_err(|error| BackendError::Process(error.to_string()))?; + let stdout = slave + .try_clone() + .map_err(|error| BackendError::Process(error.to_string()))?; + let mut command = self.command(spec)?; + command.stdin(stdin).stdout(stdout).stderr(slave); + let effective_workdir = spec.workdir.as_deref().or(self.base_workdir.as_deref()); + #[cfg(target_os = "linux")] + let prepared = self.prepare_sandbox(effective_workdir)?; + crate::ssh::unsafe_pty::install_pre_exec( + &mut command, + self.policy.clone(), + effective_workdir.map(str::to_string), + slave_fd, + self.netns_fd.as_deref().map(AsRawFd::as_raw_fd), + self.resolved_identity, + self.enforcement_mode, + #[cfg(target_os = "linux")] + prepared, + ); + #[cfg(target_os = "linux")] + let mut child_registry = crate::managed_children::lock(); + let mut child = command + .spawn() + .map_err(|error| BackendError::Process(error.to_string()))?; + let pid = child.id(); + let process_terminal = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let signal_lock = Arc::new(std::sync::Mutex::new(())); + if let Err(error) = + self.runtime + .register_process_group(pid, process_terminal.clone(), signal_lock.clone()) + { + let _ = killpg( + Pid::from_raw(i32::try_from(pid).unwrap_or(i32::MAX)), + Signal::SIGKILL, + ); + let _ = child.wait(); + return Err(error); + } + #[cfg(target_os = "linux")] + let managed_child = child_registry.register(pid); + #[cfg(target_os = "linux")] + drop(child_registry); + let terminal: Arc = Arc::new(LocalTerminal { master }); + let process = Arc::new(LocalExecProcess::new( + child, + pid, + self.runtime.clone(), + process_terminal, + signal_lock, + #[cfg(target_os = "linux")] + managed_child, + )); + Ok(SpawnedExec { + session: Some(ExecSession { + process: process.clone(), + stdin: Some(Box::new(tokio::fs::File::from_std(input))), + stdout: Box::new(tokio::fs::File::from_std(output)), + stderr: None, + terminal: Some(terminal), + }), + process, + armed: true, + }) + } +} + +struct SpawnedExec { + session: Option, + process: Arc, + armed: bool, +} + +impl SpawnedExec { + fn into_session(mut self) -> ExecSession { + self.armed = false; + self.session.take().expect("spawned exec session") + } +} + +impl Drop for SpawnedExec { + fn drop(&mut self) { + if self.armed { + let _ = self.process.deliver(Signal::SIGKILL); + } + } +} + +#[async_trait] +impl BoundaryExec for LocalBoundaryExec { + async fn exec(&self, spec: ExecSpec) -> Result { + let executor = self.clone(); + let (send, receive) = tokio::sync::oneshot::channel(); + tokio::task::spawn_blocking(move || { + let result = if spec.pty { + executor.spawn_pty(&spec) + } else { + executor.spawn_piped(&spec) + }; + // If the caller cancelled, either send fails and drops the armed + // process guard here, or the queued guard is dropped with the + // receiver. Both paths terminate an unobservable exec process. + let _ = send.send(result); + }); + receive + .await + .map_err(|_| BackendError::Process("exec spawn task failed".to_string()))? + .map(SpawnedExec::into_session) + } +} + +struct LocalTerminal { + master: std::fs::File, +} + +#[async_trait] +impl BoundaryTerminal for LocalTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + crate::ssh::unsafe_pty::set_winsize( + self.master.as_raw_fd(), + Winsize { + ws_row: rows.max(1), + ws_col: cols.max(1), + ws_xpixel: 0, + ws_ypixel: 0, + }, + ) + .map_err(|error| BackendError::Process(error.to_string())) + } +} + +struct LocalExecProcess { + pid: u32, + result: Arc>>>, + exited: Arc, + runtime: Arc, + terminal: Arc, + signal_lock: Arc>, +} + +impl LocalExecProcess { + fn new( + child: Child, + pid: u32, + runtime: Arc, + terminal: Arc, + signal_lock: Arc>, + #[cfg(target_os = "linux")] managed_child: Option, + ) -> Self { + let result = Arc::new(std::sync::Mutex::new(None)); + let exited = Arc::new(tokio::sync::Notify::new()); + let result_for_wait = result.clone(); + let exited_for_wait = exited.clone(); + let runtime_for_wait = runtime.clone(); + let terminal_for_wait = terminal.clone(); + let registration_terminal = terminal.clone(); + #[cfg(target_os = "linux")] + let signal_lock_for_wait = signal_lock.clone(); + tokio::spawn(async move { + let waited = tokio::task::spawn_blocking(move || { + let mut child = child; + #[cfg(target_os = "linux")] + { + crate::managed_children::wait_until_terminal(pid)?; + let _signal_guard = signal_lock_for_wait + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + terminal_for_wait.store(true, std::sync::atomic::Ordering::Release); + let result = child.wait(); + if let Some(managed_child) = managed_child { + crate::managed_children::unregister(managed_child); + } + result + } + #[cfg(not(target_os = "linux"))] + { + let result = child.wait(); + terminal_for_wait.store(true, std::sync::atomic::Ordering::Release); + result + } + }) + .await + .map_err(|error| error.to_string()) + .and_then(|status| status.map_err(|error| error.to_string())) + .map(|status| { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(signal) = status.signal() { + return BoundaryExitStatus::Signaled(signal); + } + } + BoundaryExitStatus::Exited(status.code().unwrap_or(1)) + }); + runtime_for_wait.unregister_process_group(pid, ®istration_terminal); + if let Ok(mut slot) = result_for_wait.lock() { + *slot = Some(waited); + } + exited_for_wait.notify_waiters(); + }); + Self { + pid, + result, + exited, + runtime, + terminal, + signal_lock, + } + } + + fn deliver(&self, signal: Signal) -> Result<(), BackendError> { + self.runtime.ensure_active()?; + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(std::sync::atomic::Ordering::Acquire) { + return Err(BackendError::Terminated("process has exited".to_string())); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + killpg(Pid::from_raw(pid), signal).map_err(|error| BackendError::Process(error.to_string())) + } +} + +#[async_trait] +impl BoundaryProcess for LocalExecProcess { + async fn wait(&self) -> Result { + loop { + let notified = self.exited.notified(); + let result = self + .result + .lock() + .map_err(|_| BackendError::Process("exec result lock poisoned".to_string()))? + .clone(); + if let Some(result) = result { + return result.map_err(BackendError::Process); + } + notified.await; + } + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + self.deliver(match signal { + BoundarySignal::Term => Signal::SIGTERM, + BoundarySignal::Kill => Signal::SIGKILL, + BoundarySignal::Int => Signal::SIGINT, + BoundarySignal::Hup => Signal::SIGHUP, + }) + } + + async fn terminate(&self) -> Result<(), BackendError> { + self.deliver(Signal::SIGKILL) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + fn executor() -> LocalBoundaryExec { + LocalBoundaryExec::new( + SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }, + None, + None, + None, + None, + ProviderCredentialState::from_environment( + 0, + HashMap::new(), + HashMap::new(), + HashMap::new(), + ), + HashMap::new(), + ResolvedProcessIdentity::default(), + ProcessEnforcementMode::NetworkOnly, + crate::boundary_io::BoundaryRuntimeState::new(), + ) + } + + #[tokio::test] + async fn non_pty_exec_preserves_stdin_stdout_and_stderr() { + let mut session = executor() + .exec(ExecSpec { + program: "/bin/sh".to_string(), + args: vec![ + "-c".to_string(), + "read line; printf 'out:%s' \"$line\"; printf 'err:%s' \"$line\" >&2" + .to_string(), + ], + env: vec![], + workdir: None, + pty: false, + }) + .await + .expect("spawn exec"); + let mut stdin = session.stdin.take().expect("stdin"); + stdin.write_all(b"value\n").await.expect("write stdin"); + drop(stdin); + let mut stdout = String::new(); + let mut stderr = String::new(); + session + .stdout + .read_to_string(&mut stdout) + .await + .expect("read stdout"); + session + .stderr + .take() + .expect("stderr") + .read_to_string(&mut stderr) + .await + .expect("read stderr"); + assert_eq!( + session.process.wait().await.unwrap(), + BoundaryExitStatus::Exited(0) + ); + assert_eq!(stdout, "out:value"); + assert_eq!(stderr, "err:value"); + } + + #[tokio::test] + async fn exec_rejects_after_boundary_end() { + let executor = executor(); + executor.runtime.deactivate(); + let result = executor + .exec(ExecSpec { + program: "/bin/true".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }) + .await; + assert!(matches!(result, Err(BackendError::Terminated(_)))); + } + + #[tokio::test] + async fn failed_exec_leaves_boundary_active_without_registered_processes() { + let executor = executor(); + let runtime = executor.runtime.clone(); + let result = executor + .exec(ExecSpec { + program: "/definitely/missing/openshell-exec".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }) + .await; + assert!(matches!(result, Err(BackendError::Process(_)))); + runtime.ensure_active().expect("boundary remains active"); + assert_eq!(runtime.registered_process_group_count(), 0); + } + + #[tokio::test] + async fn cancelled_exec_does_not_leave_a_registered_process() { + let executor = executor(); + let runtime = executor.runtime.clone(); + let task = tokio::spawn(async move { + executor + .exec(ExecSpec { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + env: vec![], + workdir: None, + pty: false, + }) + .await + }); + tokio::task::yield_now().await; + task.abort(); + let _ = task.await; + + // Give the detached blocking setup time to reach its cancelled + // handoff, including the case where cancellation won before spawn. + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while runtime.registered_process_group_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelled exec process must be terminated and reaped"); + runtime.ensure_active().expect("boundary remains active"); + } + + #[tokio::test] + async fn dropping_undelivered_exec_guard_terminates_process() { + let executor = executor(); + let runtime = executor.runtime.clone(); + let spawned = tokio::task::spawn_blocking(move || { + executor.spawn_piped(&ExecSpec { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + env: vec![], + workdir: None, + pty: false, + }) + }) + .await + .expect("spawn task") + .expect("spawn exec"); + assert_eq!(runtime.registered_process_group_count(), 1); + + // This is the post-send/pre-receive cancellation case: dropping the + // queued ownership guard must kill the process before it is observable. + drop(spawned); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while runtime.registered_process_group_count() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("undelivered exec process must be terminated and reaped"); + runtime.ensure_active().expect("boundary remains active"); + } + + #[tokio::test] + async fn completed_exec_removes_its_process_group_registration() { + let executor = executor(); + let runtime = executor.runtime.clone(); + let session = executor + .exec(ExecSpec { + program: "/bin/true".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }) + .await + .expect("spawn exec"); + assert_eq!( + session.process.wait().await.unwrap(), + BoundaryExitStatus::Exited(0) + ); + assert_eq!(runtime.registered_process_group_count(), 0); + } + + #[tokio::test] + async fn pty_exec_exposes_resize_and_stable_wait() { + let session = executor() + .exec(ExecSpec { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), "exit 7".to_string()], + env: vec![], + workdir: None, + pty: true, + }) + .await + .expect("spawn pty exec"); + session + .terminal + .as_ref() + .expect("terminal") + .resize(120, 40) + .await + .expect("resize"); + assert_eq!( + session.process.wait().await.unwrap(), + BoundaryExitStatus::Exited(7) + ); + assert_eq!( + session.process.wait().await.unwrap(), + BoundaryExitStatus::Exited(7) + ); + } +} diff --git a/crates/openshell-supervisor-process/src/boundary_io.rs b/crates/openshell-supervisor-process/src/boundary_io.rs new file mode 100644 index 0000000000..c09974d2df --- /dev/null +++ b/crates/openshell-supervisor-process/src/boundary_io.rs @@ -0,0 +1,317 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The in-pod [`BoundaryPortForward`] interface (RFC 0012 runtime contract). +//! +//! This is the live in-boundary port-forward for the in-pod placement. It lives +//! in this crate on purpose: the SSH server and supervisor session that consume +//! it are here, and so is the primitive it wraps +//! ([`connect_in_netns`](crate::ssh::connect_in_netns)). The interface trait +//! lives in the lower `openshell-isolation` crate, so this crate depends on the +//! trait (process -> isolation -> core, acyclic) and the SSH server drives a +//! `&dyn BoundaryPortForward` without depending on the backend. +//! +//! The SSH server and supervisor session are wired to this through the +//! `RunningBoundary::port_forward()` accessor: swapping in a kernel-separated +//! backend swaps this implementation (where `connect` tunnels into the guest) +//! and touches no consumer code. + +use async_trait::async_trait; +use openshell_isolation::contract::{ + BackendError, BoundaryDuplexStream, BoundaryPortForward, LoopbackTarget, +}; +use std::collections::HashMap; +use std::os::fd::{AsRawFd, OwnedFd}; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex}; + +/// Shared liveness and child-process ownership for one active boundary. +pub struct BoundaryRuntimeState { + state: AtomicU8, + process_groups: Mutex>, + exclusive_pid_namespace: bool, +} + +impl BoundaryRuntimeState { + #[must_use] + pub fn new() -> Arc { + Arc::new(Self { + state: AtomicU8::new(0), + process_groups: Mutex::new(HashMap::new()), + exclusive_pid_namespace: false, + }) + } + + /// Construct state for a boundary that exclusively owns its PID namespace. + #[must_use] + pub fn new_exclusive_pid_namespace() -> Arc { + Arc::new(Self { + state: AtomicU8::new(0), + process_groups: Mutex::new(HashMap::new()), + exclusive_pid_namespace: true, + }) + } + + #[must_use] + pub const fn requires_dedicated_process_group(&self) -> bool { + self.exclusive_pid_namespace + } + + pub fn ensure_active(&self) -> Result<(), BackendError> { + if self.state.load(Ordering::Acquire) == 0 { + Ok(()) + } else { + Err(BackendError::Terminated("boundary has ended".to_string())) + } + } + + #[must_use] + pub fn is_active(&self) -> bool { + self.state.load(Ordering::Acquire) == 0 + } + + #[must_use] + pub fn enforcement_was_lost(&self) -> bool { + self.state.load(Ordering::Acquire) == 2 + } + + pub fn register_process_group( + &self, + pid: u32, + terminal: Arc, + signal_lock: Arc>, + ) -> Result<(), BackendError> { + let mut groups = self + .process_groups + .lock() + .map_err(|_| BackendError::Process("boundary process registry poisoned".to_string()))?; + self.ensure_active()?; + groups.insert( + pid, + RegisteredProcessGroup { + pid, + terminal, + signal_lock, + }, + ); + Ok(()) + } + + pub fn unregister_process_group( + &self, + pid: u32, + terminal: &Arc, + ) { + if let Ok(mut groups) = self.process_groups.lock() + && groups + .get(&pid) + .is_some_and(|group| Arc::ptr_eq(&group.terminal, terminal)) + { + groups.remove(&pid); + } + } + + #[cfg(test)] + pub fn registered_process_group_count(&self) -> usize { + self.process_groups.lock().map_or(0, |groups| groups.len()) + } + + /// End the boundary and terminate every registered workload process group. + pub fn deactivate(&self) { + if self + .state + .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + self.terminate_registered_processes(); + } + } + + /// End the boundary because required standing enforcement was lost. + /// + /// Returns `true` only to the caller that won the active-to-terminated + /// transition. A concurrent normal teardown cannot later be reclassified + /// as enforcement loss. + pub fn deactivate_for_enforcement_loss(&self) -> bool { + if self + .state + .compare_exchange(0, 2, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return false; + } + self.terminate_registered_processes(); + true + } + + fn terminate_registered_processes(&self) { + let groups = self + .process_groups + .lock() + .map(|groups| groups.values().cloned().collect::>()) + .unwrap_or_default(); + for group in groups { + group.terminate(); + } + } +} + +#[derive(Clone)] +struct RegisteredProcessGroup { + pid: u32, + terminal: Arc, + signal_lock: Arc>, +} + +impl RegisteredProcessGroup { + fn terminate(&self) { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return; + } + if let Ok(pid) = i32::try_from(self.pid) { + let _ = nix::sys::signal::killpg( + nix::unistd::Pid::from_raw(pid), + nix::sys::signal::Signal::SIGKILL, + ); + } + } +} + +/// In-pod loopback port-forward: connects to a loopback target from inside the +/// workload's network namespace via [`connect_in_netns`](crate::ssh::connect_in_netns). +pub struct NetnsPortForward { + /// File descriptor of the boundary's network namespace, or `None` to + /// connect from the supervisor's own namespace. + netns_fd: Option>, + runtime: Option>, +} + +impl NetnsPortForward { + #[must_use] + pub fn new(netns_fd: Option>, runtime: Option>) -> Self { + Self { netns_fd, runtime } + } +} + +#[async_trait] +impl BoundaryPortForward for NetnsPortForward { + async fn connect(&self, target: LoopbackTarget) -> Result { + if let Some(runtime) = &self.runtime { + runtime.ensure_active()?; + } + let addr = std::net::SocketAddr::new(target.host(), target.port()); + let addr_string = addr.to_string(); + let stream = crate::ssh::connect_in_netns( + &addr_string, + self.netns_fd.as_deref().map(AsRawFd::as_raw_fd), + ) + .await + .map_err(|e| BackendError::Process(format!("port-forward connect to {addr}: {e}")))?; + if let Some(runtime) = &self.runtime { + runtime.ensure_active()?; + } + Ok(Box::new(stream)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::Ipv4Addr; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + /// Stands in for the SSH server's port-forward path: connect through the + /// interface, write, read the echo. With `netns_fd: None` the connect happens in + /// the supervisor's namespace, so this exercises the real primitive without + /// requiring a network namespace. + #[tokio::test] + async fn port_forward_connects_and_round_trips() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut sock, _) = listener.accept().await.unwrap(); + let mut buf = [0u8; 4]; + sock.read_exact(&mut buf).await.unwrap(); + sock.write_all(&buf).await.unwrap(); + }); + + let pf = NetnsPortForward::new(None, None); + let target = + LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), addr.port()).expect("loopback target"); + let mut conn = pf.connect(target).await.expect("connect through interface"); + conn.write_all(b"ping").await.unwrap(); + let mut buf = [0u8; 4]; + conn.read_exact(&mut buf).await.unwrap(); + assert_eq!(&buf, b"ping"); + } + + /// Drive the port-forward interface through a generic `&dyn` consumer, proving a + /// kernel-separated backend (tunneling into a guest) would use the same call. + #[tokio::test] + async fn port_forward_is_driven_via_dyn() { + async fn forward_one(pf: &dyn BoundaryPortForward, target: LoopbackTarget) -> bool { + pf.connect(target).await.is_ok() + } + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let _ = listener.accept().await; + }); + let pf = NetnsPortForward::new(None, None); + let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), addr.port()).unwrap(); + assert!(forward_one(&pf, target).await); + } + + #[tokio::test] + async fn port_forward_rejects_after_boundary_end() { + let runtime = BoundaryRuntimeState::new(); + let pf = NetnsPortForward::new(None, Some(runtime.clone())); + runtime.deactivate(); + let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), 1).unwrap(); + assert!(matches!( + pf.connect(target).await, + Err(BackendError::Terminated(_)) + )); + } + + #[tokio::test] + async fn failed_port_forward_keeps_boundary_active() { + let listener = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let runtime = BoundaryRuntimeState::new(); + let pf = NetnsPortForward::new(None, Some(runtime.clone())); + let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), port).unwrap(); + assert!(matches!( + pf.connect(target).await, + Err(BackendError::Process(_)) + )); + runtime.ensure_active().expect("boundary remains active"); + } + + #[test] + fn stale_unregister_preserves_reused_process_group_registration() { + let runtime = BoundaryRuntimeState::new(); + let first_terminal = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let second_terminal = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let pid = 42; + runtime + .register_process_group(pid, first_terminal.clone(), Arc::new(Mutex::new(()))) + .expect("first registration"); + runtime + .register_process_group(pid, second_terminal.clone(), Arc::new(Mutex::new(()))) + .expect("replacement registration"); + + runtime.unregister_process_group(pid, &first_terminal); + assert_eq!(runtime.registered_process_group_count(), 1); + + runtime.unregister_process_group(pid, &second_terminal); + assert_eq!(runtime.registered_process_group_count(), 0); + } +} diff --git a/crates/openshell-supervisor-process/src/lib.rs b/crates/openshell-supervisor-process/src/lib.rs index 743942faa4..ee6bedeb22 100644 --- a/crates/openshell-supervisor-process/src/lib.rs +++ b/crates/openshell-supervisor-process/src/lib.rs @@ -8,6 +8,8 @@ //! and log push. Populated by follow-up commits as modules migrate out of //! `openshell-sandbox`. +pub mod boundary_exec; +pub mod boundary_io; pub mod child_env; pub mod debug_rpc; #[cfg(unix)] diff --git a/crates/openshell-supervisor-process/src/managed_children.rs b/crates/openshell-supervisor-process/src/managed_children.rs index 311c80693f..04f4114a04 100644 --- a/crates/openshell-supervisor-process/src/managed_children.rs +++ b/crates/openshell-supervisor-process/src/managed_children.rs @@ -10,44 +10,146 @@ #![cfg(target_os = "linux")] -use std::collections::HashSet; -use std::sync::{LazyLock, Mutex}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{LazyLock, Mutex, MutexGuard}; -static MANAGED_CHILDREN: LazyLock>> = - LazyLock::new(|| Mutex::new(HashSet::new())); +static MANAGED_CHILDREN: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); +static NEXT_GENERATION: AtomicU64 = AtomicU64::new(1); -/// Add `pid` to the supervised-child set. Non-positive or out-of-range values -/// are silently ignored. -pub fn register(pid: u32) { - let Ok(pid) = i32::try_from(pid) else { - return; - }; - if pid <= 0 { - return; +/// Identity of one registry entry. The generation prevents an old waiter from +/// removing a newer child that reused the same numeric PID after reap. +#[derive(Clone, Copy)] +pub struct ManagedChild { + pid: i32, + generation: u64, +} + +/// A managed-child registration accepted by [`unregister`]. +/// +/// New boundary-owned processes retain a generation-bearing token. Legacy +/// supervisor paths still identify their child by PID; supporting both keeps +/// the registry race-safe for new code without forcing an unrelated rewrite +/// of the canonical main-process and SSH paths. +pub enum ManagedChildRegistration { + Token(ManagedChild), + Pid(u32), +} + +impl From for ManagedChildRegistration { + fn from(value: ManagedChild) -> Self { + Self::Token(value) } - if let Ok(mut children) = MANAGED_CHILDREN.lock() { - children.insert(pid); +} + +impl From for ManagedChildRegistration { + fn from(value: u32) -> Self { + Self::Pid(value) } } -/// Remove `pid` from the supervised-child set. Non-positive or out-of-range -/// values are silently ignored. -pub fn unregister(pid: u32) { - let Ok(pid) = i32::try_from(pid) else { - return; - }; - if pid <= 0 { - return; +/// Exclusive access to the managed-child registry. +/// +/// A process spawner holds this guard from immediately before `spawn` or +/// `fork` until the returned PID is registered. The orphan reaper holds the +/// same guard while deciding whether to reap an exited child. This closes the +/// otherwise unavoidable window in which a fast-exiting managed child exists +/// but its PID has not yet been published. +pub struct RegistryGuard(MutexGuard<'static, HashMap>); + +impl RegistryGuard { + /// Add a newly spawned managed child. + pub fn register(&mut self, pid: u32) -> Option { + let Ok(pid) = i32::try_from(pid) else { + return None; + }; + if pid <= 0 { + return None; + } + let generation = NEXT_GENERATION.fetch_add(1, Ordering::Relaxed); + self.0.insert(pid, generation); + Some(ManagedChild { pid, generation }) + } + + /// Return whether the PID belongs to an explicit waiter. + #[must_use] + pub fn contains(&self, pid: i32) -> bool { + self.0.contains_key(&pid) } +} + +/// Lock the registry for an atomic spawn-and-register or inspect-and-reap +/// operation. +pub fn lock() -> RegistryGuard { + RegistryGuard( + MANAGED_CHILDREN + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ) +} + +/// Register a child for a legacy caller that cannot retain a generation token. +pub fn register(pid: u32) { + let _ = lock().register(pid); +} + +/// Remove exactly this supervised-child registration. A newer registration +/// for a reused PID is preserved. +pub fn unregister(child: impl Into) { if let Ok(mut children) = MANAGED_CHILDREN.lock() { - children.remove(&pid); + match child.into() { + ManagedChildRegistration::Token(child) + if children.get(&child.pid) == Some(&child.generation) => + { + children.remove(&child.pid); + } + ManagedChildRegistration::Pid(pid) => { + if let Ok(pid) = i32::try_from(pid) { + children.remove(&pid); + } + } + ManagedChildRegistration::Token(_) => {} + } } } /// Return `true` if `pid` is currently in the supervised-child set. #[must_use] pub fn is_managed(pid: i32) -> bool { - MANAGED_CHILDREN - .lock() - .is_ok_and(|children| children.contains(&pid)) + lock().contains(pid) +} + +/// Wait until a managed child is terminal without reaping it. +/// +/// Keeping the child as a zombie prevents PID/process-group reuse until the +/// owner publishes terminal state and performs the final wait. +pub fn wait_until_terminal(pid: u32) -> std::io::Result<()> { + use nix::sys::wait::{Id, WaitPidFlag, waitid}; + let pid = i32::try_from(pid) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidInput, "PID out of range"))?; + waitid( + Id::Pid(nix::unistd::Pid::from_raw(pid)), + WaitPidFlag::WEXITED | WaitPidFlag::WNOWAIT, + ) + .map(|_| ()) + .map_err(std::io::Error::other) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stale_unregister_preserves_reused_pid_registration() { + let pid = i32::MAX as u32; + let first = lock().register(pid).expect("first registration"); + let second = lock().register(pid).expect("replacement registration"); + + unregister(first); + assert!(is_managed(i32::try_from(pid).expect("test pid"))); + + unregister(second); + assert!(!is_managed(i32::try_from(pid).expect("test pid"))); + } } diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index aef95b6068..2fb075b420 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -24,7 +24,7 @@ pub struct NftCommand { pub required: bool, } -/// Generate nft commands for sandbox network bypass enforcement. +/// Generate the legacy nft commands for sandbox bypass detection. /// /// Creates an `inet` family table (handles both IPv4 and IPv6) with rules that: /// 1. Accept traffic to the proxy (IPv4 only) @@ -34,11 +34,34 @@ pub struct NftCommand { /// /// If `log_prefix` is provided, log rules are inserted before each reject rule /// so that bypass attempts are recorded in the kernel ring buffer before being -/// rejected. Log rules are always non-required since they need `nf_log` support. +/// rejected. Log rules are non-required since they need `nf_log` support. pub fn generate_bypass_commands( host_ip: &str, proxy_port: u16, log_prefix: Option<&str>, +) -> Vec { + generate_commands(host_ip, proxy_port, log_prefix, false) +} + +/// Generate the RFC 0012 default-deny egress ceiling. +/// +/// Only the exact proxy destination and loopback are accepted. TCP and UDP +/// rejects are optional fast-fail behavior; the base-chain drop policy covers +/// every address family and protocol. No blanket conntrack exception is +/// installed because pre-existing or related flows must not bypass mediation. +pub fn generate_egress_ceiling_commands( + host_ip: &str, + proxy_port: u16, + log_prefix: Option<&str>, +) -> Vec { + generate_commands(host_ip, proxy_port, log_prefix, true) +} + +fn generate_commands( + host_ip: &str, + proxy_port: u16, + log_prefix: Option<&str>, + default_deny: bool, ) -> Vec { let table = "openshell_bypass"; let mut cmds = vec![ @@ -52,7 +75,11 @@ pub fn generate_bypass_commands( "inet", table, "output", - "{ type filter hook output priority 0; policy accept; }", + if default_deny { + "{ type filter hook output priority 0; policy drop; }" + } else { + "{ type filter hook output priority 0; policy accept; }" + }, ], ), nft_cmd( @@ -78,7 +105,10 @@ pub fn generate_bypass_commands( "add", "rule", "inet", table, "output", "oifname", "lo", "accept", ], ), - nft_cmd( + ]; + + if !default_deny { + cmds.push(nft_cmd( false, &[ "add", @@ -91,8 +121,8 @@ pub fn generate_bypass_commands( "established,related", "accept", ], - ), - ]; + )); + } if let Some(prefix) = log_prefix { let quoted = nft_quote(prefix); @@ -106,7 +136,7 @@ pub fn generate_bypass_commands( } cmds.push(nft_cmd( - true, + !default_deny, &[ "add", "rule", @@ -127,7 +157,7 @@ pub fn generate_bypass_commands( ], )); cmds.push(nft_cmd( - true, + !default_deny, &[ "add", "rule", @@ -160,7 +190,7 @@ pub fn generate_bypass_commands( } cmds.push(nft_cmd( - true, + !default_deny, &[ "add", "rule", @@ -181,7 +211,7 @@ pub fn generate_bypass_commands( ], )); cmds.push(nft_cmd( - true, + !default_deny, &[ "add", "rule", @@ -598,6 +628,25 @@ mod tests { assert!(text.contains("type filter hook output priority 0; policy accept;")); } + #[test] + fn in_pod_ceiling_is_default_deny_for_all_protocols() { + let text = all_strs(&generate_egress_ceiling_commands("10.0.2.2", 3128, None)); + assert!(text.contains("policy drop")); + assert!(!text.contains("policy accept")); + assert!(!text.contains("ct state")); + } + + #[test] + fn in_pod_reject_rules_are_optional_fast_fail_over_default_drop() { + let commands = generate_egress_ceiling_commands("10.0.2.2", 3128, None); + for command in commands + .iter() + .filter(|command| command.args.iter().any(|argument| argument == "reject")) + { + assert!(!command.required); + } + } + #[test] fn proxy_accept_rule_uses_provided_ip_and_port() { let cmds = generate_bypass_commands("172.16.0.1", 9999, None); @@ -611,7 +660,7 @@ mod tests { let text = all_strs(&cmds); let proxy_pos = text.find("ip daddr").unwrap(); let lo_pos = text.find("oifname lo").unwrap(); - let ct_pos = text.find("ct state established,related").unwrap(); + let ct_pos = text.find("ct state established").unwrap(); let reject_pos = text.find("reject with icmp type").unwrap(); assert!(proxy_pos < lo_pos); diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs b/crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs index a9c67af95a..ddd37a502d 100644 --- a/crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs +++ b/crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs @@ -838,4 +838,43 @@ mod tests { "socket(AF_NETLINK, SOCK_RAW, NETLINK_SOCK_DIAG) should be blocked with EPERM" ); } + + #[test] + fn behavioral_block_mode_denies_inet_and_packet_sockets() { + let filter = build_filter(false).unwrap(); + let pid = unsafe { libc::fork() }; + assert!(pid >= 0, "fork failed"); + if pid == 0 { + unsafe { + libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + apply_filter(&filter).expect("apply block-mode filter"); + for (domain, socket_type, protocol) in [ + (libc::AF_INET, libc::SOCK_STREAM, 0), + (libc::AF_INET6, libc::SOCK_DGRAM, 0), + (libc::AF_PACKET, libc::SOCK_RAW, 0), + ] { + let fd = libc::socket(domain, socket_type, protocol); + let errno = *libc::__errno_location(); + if fd >= 0 || errno != libc::EPERM { + if fd >= 0 { + libc::close(fd); + } + libc::_exit(1); + } + } + let unix_fd = libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0); + if unix_fd < 0 { + libc::_exit(1); + } + libc::close(unix_fd); + libc::_exit(0); + } + } + let mut status: libc::c_int = 0; + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!( + unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, + "block mode must deny IPv4, IPv6, and packet sockets while retaining Unix IPC" + ); + } } diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 893967b2ac..c0b5a2c30f 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -1134,7 +1134,7 @@ impl Default for PtyRequest { } #[allow(clippy::too_many_arguments)] -fn apply_child_env( +pub(crate) fn apply_child_env( cmd: &mut Command, session_home: &str, session_user: &str, @@ -1530,7 +1530,7 @@ fn spawn_pipe_exec( Ok(sender) } -mod unsafe_pty { +pub(crate) mod unsafe_pty { #[cfg(not(target_os = "linux"))] use super::sandbox; use super::{ From 91dd3cd65a651c77d032a2bef737138381511a6f Mon Sep 17 00:00:00 2001 From: Drew Newberry <385+drew@users.noreply.github.com> Date: Wed, 26 Aug 2026 04:56:26 +0000 Subject: [PATCH 3/3] feat(isolation): enable the co-located RFC 0012 backend Signed-off-by: Drew Newberry <385+drew@users.noreply.github.com> --- .agents/skills/helm-dev-environment/SKILL.md | 3 + .github/workflows/branch-checks.yml | 55 + .github/workflows/driver-vm-linux.yml | 14 +- .github/workflows/driver-vm-macos.yml | 19 +- Cargo.lock | 17 + architecture/sandbox.md | 28 + crates/openshell-core/src/driver_mounts.rs | 18 + crates/openshell-driver-docker/Cargo.toml | 5 +- crates/openshell-driver-docker/src/lib.rs | 194 +- crates/openshell-driver-docker/src/tests.rs | 23 +- crates/openshell-driver-kubernetes/Cargo.toml | 1 + .../openshell-driver-kubernetes/src/driver.rs | 263 ++- crates/openshell-driver-podman/Cargo.toml | 1 + .../openshell-driver-podman/src/container.rs | 33 +- crates/openshell-driver-vm/Cargo.toml | 1 + crates/openshell-driver-vm/README.md | 6 +- crates/openshell-driver-vm/build.rs | 16 +- crates/openshell-driver-vm/runtime/README.md | 2 +- .../scripts/openshell-vm-sandbox-init.sh | 32 +- crates/openshell-driver-vm/src/driver.rs | 6 +- crates/openshell-driver-vm/src/rootfs.rs | 146 +- crates/openshell-sandbox/Cargo.toml | 5 +- crates/openshell-sandbox/src/inpod.rs | 1622 +++++++++++++++++ crates/openshell-sandbox/src/lib.rs | 408 +++-- crates/openshell-sandbox/src/main.rs | 35 + .../openshell-supervisor-network/src/proxy.rs | 444 ++++- .../openshell-supervisor-network/src/run.rs | 3 + .../src/netns/mod.rs | 142 ++ .../src/process.rs | 10 + .../openshell-supervisor-process/src/run.rs | 339 +++- deploy/docker/Dockerfile.supervisor | 23 +- docs/reference/gateway-config.mdx | 13 +- tasks/scripts/gateway-vm.sh | 3 +- tasks/scripts/vm/build-supervisor-bundle.sh | 46 + 34 files changed, 3630 insertions(+), 346 deletions(-) create mode 100644 crates/openshell-sandbox/src/inpod.rs diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 2dad568c79..c4579ba523 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -65,6 +65,9 @@ mise run helm:skaffold:run mise run helm:skaffold:run:sidecar ``` +Combined topology selects RFC 0012's in-pod backend and supplies its descriptor; +sidecar topology remains on its separate lifecycle. + **Supervisor sidecar topology with TLS/mTLS enabled** (build once and leave running): ```bash mise run helm:skaffold:run:sidecar-mtls diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index bece5c8825..425108d913 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -161,6 +161,61 @@ jobs: fi exit 0 + isolation-conformance: + name: Isolation conformance (privileged Linux) + needs: pr_metadata + if: needs.pr_metadata.outputs.should_run == 'true' + runs-on: linux-amd64-cpu8 + timeout-minutes: 20 + container: + image: ghcr.io/nvidia/openshell/ci:latest + options: --privileged + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install tools and network helpers + run: | + apt-get update + apt-get install -y --no-install-recommends iproute2 nftables iptables util-linux + mise install --locked + + - name: Materialize the Alpine trusted helper runtime fixture + run: | + alpine_root="${RUNNER_TEMP}/openshell-alpine-root" + runtime="${RUNNER_TEMP}/openshell-runtime" + archive="${RUNNER_TEMP}/alpine-minirootfs.tar.gz" + curl -fsSL \ + https://dl-cdn.alpinelinux.org/alpine/v3.22/releases/x86_64/alpine-minirootfs-3.22.5-x86_64.tar.gz \ + -o "$archive" + echo "4b4daa9fe2fc696c4919c4412a4c3d3e770d8fb70292a004a2c72f5096175282 $archive" \ + | sha256sum -c - + mkdir -p "$alpine_root" "$runtime" + tar -xzf "$archive" -C "$alpine_root" + cp /etc/resolv.conf "$alpine_root/etc/resolv.conf" + chroot "$alpine_root" /sbin/apk add --no-cache \ + iproute2 nftables iptables iptables-legacy + for path in /bin /sbin /lib /lib64 /usr/bin /usr/sbin /usr/lib /usr/lib64 /etc/iproute2 /usr/share/nftables; do + if [ -e "$alpine_root$path" ]; then + (cd "$alpine_root" && cp -aL --parents ".$path" "$runtime") + fi + done + chmod -R go-w "$runtime" + + - name: Exercise the live default-deny ceiling + run: | + cargo test -p openshell-isolation -p openshell-supervisor-process \ + -p openshell-supervisor-network -p openshell-sandbox + OPENSHELL_TEST_TRUSTED_RUNTIME_ROOT="${RUNNER_TEMP}/openshell-runtime" \ + cargo test -p openshell-supervisor-process \ + installed_egress_ceiling_ -- \ + --ignored --nocapture --test-threads=1 + cargo test -p openshell-sandbox \ + pid_one_exit_kills_unregistered_setsid_descendant_within_bound -- \ + --ignored --nocapture --test-threads=1 + rust-macos: name: Rust lint (macOS) needs: pr_metadata diff --git a/.github/workflows/driver-vm-linux.yml b/.github/workflows/driver-vm-linux.yml index 942cacdfbd..355253c148 100644 --- a/.github/workflows/driver-vm-linux.yml +++ b/.github/workflows/driver-vm-linux.yml @@ -116,6 +116,8 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} @@ -141,8 +143,12 @@ jobs: cache-directories: .cache/sccache cache-targets: "true" - - name: Install zstd - run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* + - name: Install zstd and verify Docker + run: | + apt-get update + apt-get install -y --no-install-recommends zstd + rm -rf /var/lib/apt/lists/* + docker info - name: Download kernel runtime tarball uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -168,14 +174,14 @@ jobs: - name: Verify embedded driver inputs run: | set -euo pipefail - for file in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst umoci.zst openshell-sandbox.zst; do + for file in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst umoci.zst openshell-sandbox.zst openshell-runtime.tar.zst; do test -s "target/vm-runtime-compressed/${file}" done - name: Scope workspace to driver-vm crates run: | set -euo pipefail - sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core"]|' Cargo.toml + sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core", "crates/openshell-isolation"]|' Cargo.toml - name: Patch workspace version if: ${{ inputs['cargo-version'] != '' }} diff --git a/.github/workflows/driver-vm-macos.yml b/.github/workflows/driver-vm-macos.yml index a97ade9cbb..41636eaad1 100644 --- a/.github/workflows/driver-vm-macos.yml +++ b/.github/workflows/driver-vm-macos.yml @@ -75,6 +75,8 @@ jobs: credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + volumes: + - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} @@ -100,8 +102,12 @@ jobs: cache-directories: .cache/sccache cache-targets: "true" - - name: Install zstd - run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* + - name: Install zstd and verify Docker + run: | + apt-get update + apt-get install -y --no-install-recommends zstd + rm -rf /var/lib/apt/lists/* + docker info - name: Build bundled supervisor run: | @@ -116,7 +122,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: driver-vm-supervisor-arm64 - path: target/vm-runtime-compressed/openshell-sandbox.zst + path: | + target/vm-runtime-compressed/openshell-sandbox.zst + target/vm-runtime-compressed/openshell-runtime.tar.zst retention-days: 1 build-driver-vm-macos: @@ -180,12 +188,13 @@ jobs: run: | set -euo pipefail test -f target/vm-runtime-compressed-macos/openshell-sandbox.zst - ls -lh target/vm-runtime-compressed-macos/openshell-sandbox.zst + test -f target/vm-runtime-compressed-macos/openshell-runtime.tar.zst + ls -lh target/vm-runtime-compressed-macos/openshell-{sandbox,runtime.tar}.zst - name: Verify embedded driver inputs run: | set -euo pipefail - for file in libkrun.dylib.zst libkrunfw.5.dylib.zst gvproxy.zst umoci.zst openshell-sandbox.zst; do + for file in libkrun.dylib.zst libkrunfw.5.dylib.zst gvproxy.zst umoci.zst openshell-sandbox.zst openshell-runtime.tar.zst; do test -s "target/vm-runtime-compressed-macos/${file}" done diff --git a/Cargo.lock b/Cargo.lock index b9d4b7a32f..347d41f55f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3401,6 +3401,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.3" @@ -3573,6 +3582,7 @@ dependencies = [ "cfg-if", "cfg_aliases", "libc", + "memoffset", ] [[package]] @@ -3948,6 +3958,7 @@ dependencies = [ "futures", "miette", "openshell-core", + "openshell-isolation", "openshell-otel", "openshell-otel-test-support", "opentelemetry", @@ -3980,6 +3991,7 @@ dependencies = [ "miette", "notify", "openshell-core", + "openshell-isolation", "openshell-policy", "prost", "prost-types", @@ -4027,6 +4039,7 @@ dependencies = [ "miette", "nix 0.29.0", "openshell-core", + "openshell-isolation", "openshell-otel", "openshell-otel-test-support", "opentelemetry", @@ -4083,6 +4096,7 @@ dependencies = [ "nix 0.29.0", "oci-client", "openshell-core", + "openshell-isolation", "openshell-otel", "openshell-otel-test-support", "openshell-policy", @@ -4257,12 +4271,15 @@ dependencies = [ name = "openshell-sandbox" version = "0.0.0" dependencies = [ + "async-trait", + "base64 0.22.1", "clap", "futures", "miette", "nix 0.29.0", "openshell-core", "openshell-extension-core", + "openshell-isolation", "openshell-ocsf", "openshell-policy", "openshell-supervisor-middleware", diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 786ed5194d..3473974920 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -50,6 +50,34 @@ OpenShell uses overlapping controls rather than a single sandbox primitive: The supervisor may enrich baseline filesystem allowances for runtime-required paths, such as proxy support files or GPU device paths when a GPU is present. +## Isolation Backend + +[RFC 0012](../rfc/0012-isolation-backend/README.md) defines the Isolation +Backend contract for topology-specific boundary construction and process +operations. The contract uses consuming +lifecycle states (`attach` → `Bound` → `confirm` → `Ready` → `start_agent` → +`Running`) so untrusted workload execution cannot begin before standing +enforcement is confirmed. + +The logical supervisor remains the trusted bridge between the gateway and the +workload. It drives the backend and applies approved network policy through +supervisor-owned mediation; the backend routes workload egress to that +mediation. The trusted Kubernetes driver selects the co-located backend for +combined topology and supplies its topology descriptor to the supervisor. +Sidecar topology remains on its pre-RFC lifecycle; a conforming backend for +that placement requires separate design and implementation. Docker, Podman, +and VM drivers provision the same co-located topology and supply its descriptor +by default. The co-located backend requires the +supervisor to own the execution environment's PID namespace so boundary +teardown can terminate every remaining workload process. + +For proxy-mode boundaries, the co-located backend verifies its default-deny +kernel egress ceiling before exposing any workload execution surface and then +rechecks it every 250 milliseconds. Each check has a two-second deadline. +Verification failure or timeout ends the boundary and triggers process cleanup; +the PID-1 supervisor exits so the kernel terminates the complete workload PID +namespace. The topology's detection-and-termination bound is five seconds. + ## Network and Inference See [Sandbox Limits](sandbox-limits.md) for the current numeric safety ceilings, diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index b1a3049882..235f1f21f3 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -343,6 +343,24 @@ mod tests { assert!(err.contains("/etc/openshell")); } + #[test] + fn container_target_rejects_parents_that_shadow_reserved_trees() { + for target in ["/opt", "/etc", "/run"] { + let err = validate_container_mount_target(target).unwrap_err(); + assert!( + err.contains("reserved OpenShell path"), + "expected {target} to be rejected: {err}" + ); + } + } + + #[test] + fn container_target_rejects_proc_shadowing() { + for target in ["/proc", "/proc/self", "/"] { + assert!(validate_container_mount_target(target).is_err()); + } + } + #[test] fn container_target_does_not_prefix_match_unrelated_paths() { validate_container_mount_target("/etc/openshell-tools").unwrap(); diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 1c9e675f77..23c0b97f37 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-isolation = { path = "../openshell-isolation" } openshell-otel = { path = "../openshell-otel" } opentelemetry = { workspace = true } @@ -35,15 +36,15 @@ url = { workspace = true } clap = { workspace = true } miette = { workspace = true } toml = { workspace = true } +tar = "0.4" +tempfile = "3" [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } prost-types = { workspace = true } -tar = "0.4" temp-env = "0.3" -tempfile = "3" tracing-subscriber = { workspace = true } [lints] diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index b1859f54cc..50ea3d3315 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -79,6 +79,8 @@ const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; +const SUPERVISOR_RUNTIME_MOUNT_PATH: &str = "/opt/openshell/bin/openshell-runtime"; +const SUPERVISOR_IMAGE_RUNTIME_PATH: &str = "/openshell-runtime"; const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; @@ -134,9 +136,9 @@ pub struct DockerComputeConfig { /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. pub supervisor_bin: Option, - /// Optional image used to extract the Linux `openshell-sandbox` binary. - /// Ignored when `supervisor_bin` is set. See `resolve_supervisor_bin` for - /// the full resolution order. + /// Optional image used to extract the Linux `openshell-sandbox` binary and + /// its trusted helper runtime. With `supervisor_bin`, the image remains the + /// runtime source unless the binary has a valid sibling runtime directory. pub supervisor_image: Option, /// Host-side CA certificate for Docker sandbox mTLS. @@ -210,6 +212,7 @@ struct DockerDriverRuntimeConfig { stop_timeout_secs: u32, log_level: String, supervisor_bin: PathBuf, + supervisor_runtime: PathBuf, guest_tls: Option, daemon_version: String, supports_gpu: bool, @@ -589,6 +592,8 @@ impl DockerComputeDriver { ); let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; + let supervisor_runtime = + resolve_supervisor_runtime(&docker, &docker_config, &supervisor_bin).await?; let guest_tls = docker_guest_tls_paths(&docker_config)?; let driver = Self { @@ -605,6 +610,7 @@ impl DockerComputeDriver { stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: config.log_level.clone(), supervisor_bin, + supervisor_runtime, guest_tls, daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), supports_gpu, @@ -2609,11 +2615,18 @@ fn build_binds( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, ) -> Result, Status> { - let mut binds = vec![format!( - "{}:{}:ro,z", - config.supervisor_bin.display(), - SUPERVISOR_MOUNT_PATH - )]; + let mut binds = vec![ + format!( + "{}:{}:ro,z", + config.supervisor_bin.display(), + SUPERVISOR_MOUNT_PATH + ), + format!( + "{}:{}:ro,z", + config.supervisor_runtime.display(), + SUPERVISOR_RUNTIME_MOUNT_PATH + ), + ]; if let Some(tls) = &config.guest_tls { binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); binds.push(format!( @@ -3025,8 +3038,18 @@ fn build_container_create_body_for_image( env: Some(build_environment_for_oci_user(sandbox, config, &image.user)), entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), // Replace the image CMD with the supervisor's resolved workspace - // argument so Docker cannot append inherited image arguments. - cmd: Some(vec!["--workdir".to_string(), workspace_root]), + // and admitted topology arguments so Docker cannot append inherited + // image arguments or select the security boundary from image state. + cmd: Some(vec![ + "--workdir".to_string(), + workspace_root, + "--topology-backend-name=in-pod".to_string(), + format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ), + "--topology-payload-base64=".to_string(), + ]), labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, @@ -3764,6 +3787,32 @@ pub(crate) async fn resolve_supervisor_bin( } } +async fn resolve_supervisor_runtime( + docker: &Docker, + docker_config: &DockerComputeConfig, + supervisor_bin: &Path, +) -> CoreResult { + if let Some(runtime) = supervisor_bin + .parent() + .map(|parent| parent.join("openshell-runtime")) + && validate_supervisor_runtime(&runtime).is_ok() + { + return Ok(runtime); + } + + let image = docker_config + .supervisor_image + .clone() + .unwrap_or_else(openshell_core::config::default_supervisor_image); + let extracted_bin = extract_supervisor_bin_from_image(docker, &image).await?; + let runtime = extracted_bin + .parent() + .expect("cache path has a parent") + .join("openshell-runtime"); + validate_supervisor_runtime(&runtime)?; + Ok(runtime) +} + fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { match daemon_arch { "arm64" => vec![PathBuf::from( @@ -3836,6 +3885,8 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core .map_err(Error::config)?; if cache_path.is_file() { validate_linux_elf_binary(&cache_path).map_err(Error::config)?; + ensure_cached_supervisor_runtime(docker, image, cache_path.parent().expect("cache parent")) + .await?; return Ok(cache_path); } @@ -3849,9 +3900,96 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; write_cache_binary_atomic(&cache_path, &binary_bytes).map_err(Error::config)?; validate_linux_elf_binary(&cache_path).map_err(Error::config)?; + ensure_cached_supervisor_runtime(docker, image, cache_path.parent().expect("cache parent")) + .await?; Ok(cache_path) } +fn validate_supervisor_runtime(runtime: &Path) -> CoreResult<()> { + if !runtime.is_dir() { + return Err(Error::config(format!( + "trusted supervisor helper runtime '{}' is missing", + runtime.display() + ))); + } + let has_ip = ["usr/sbin/ip", "sbin/ip", "usr/bin/ip", "bin/ip"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_nft = ["usr/sbin/nft", "sbin/nft", "usr/bin/nft"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_loader = std::fs::read_dir(runtime.join("lib")) + .ok() + .into_iter() + .flatten() + .filter_map(Result::ok) + .any(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("ld-musl-") && name.ends_with(".so.1")) + }); + if !has_ip || !has_nft || !has_loader { + return Err(Error::config(format!( + "trusted supervisor helper runtime '{}' is incomplete", + runtime.display() + ))); + } + Ok(()) +} + +async fn ensure_cached_supervisor_runtime( + docker: &Docker, + image: &str, + cache_dir: &Path, +) -> CoreResult<()> { + let runtime = cache_dir.join("openshell-runtime"); + if validate_supervisor_runtime(&runtime).is_ok() { + return Ok(()); + } + + let archive = extract_supervisor_runtime_archive(docker, image).await?; + let staging = tempfile::Builder::new() + .prefix(".openshell-runtime-") + .tempdir_in(cache_dir) + .map_err(|err| Error::config(format!("create runtime staging directory: {err}")))?; + let mut tar = tar::Archive::new(std::io::Cursor::new(archive)); + for entry in tar + .entries() + .map_err(|err| Error::config(format!("open supervisor runtime archive: {err}")))? + { + let mut entry = entry + .map_err(|err| Error::config(format!("read supervisor runtime archive: {err}")))?; + let kind = entry.header().entry_type(); + if !kind.is_file() && !kind.is_dir() { + return Err(Error::config( + "supervisor runtime archive contains a non-materialized link or special file", + )); + } + if !entry + .unpack_in(staging.path()) + .map_err(|err| Error::config(format!("extract supervisor runtime archive: {err}")))? + { + return Err(Error::config( + "supervisor runtime archive contains a path outside its root", + )); + } + } + let extracted = staging.path().join("openshell-runtime"); + validate_supervisor_runtime(&extracted)?; + match std::fs::rename(&extracted, &runtime) { + Ok(()) => {} + Err(_) if validate_supervisor_runtime(&runtime).is_ok() => {} + Err(err) => { + return Err(Error::config(format!( + "install trusted supervisor runtime '{}': {err}", + runtime.display() + ))); + } + } + validate_supervisor_runtime(&runtime) +} + async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { let mut stream = docker.create_image( Some(CreateImageOptions { @@ -3875,6 +4013,19 @@ async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { /// binary as a tar archive, and return the untarred file bytes. The /// container is always removed, even on error paths. async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { + extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_BINARY_PATH, true).await +} + +async fn extract_supervisor_runtime_archive(docker: &Docker, image: &str) -> CoreResult> { + extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_RUNTIME_PATH, false).await +} + +async fn extract_supervisor_path_archive( + docker: &Docker, + image: &str, + path: &str, + extract_single_file: bool, +) -> CoreResult> { let container_name = temp_extract_container_name(); docker .create_container( @@ -3898,7 +4049,8 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe })?; // Always tear down the extractor container, even if extraction fails. - let result = download_binary_from_container(docker, &container_name).await; + let result = + download_path_from_container(docker, &container_name, path, extract_single_file).await; if let Err(remove_err) = docker .remove_container( &container_name, @@ -3915,12 +4067,14 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe result } -async fn download_binary_from_container( +async fn download_path_from_container( docker: &Docker, container_name: &str, + path: &str, + extract_single_file: bool, ) -> CoreResult> { let options = DownloadFromContainerOptionsBuilder::default() - .path(SUPERVISOR_IMAGE_BINARY_PATH) + .path(path) .build(); let mut stream = docker.download_from_container(container_name, Some(options)); @@ -3934,11 +4088,15 @@ async fn download_binary_from_container( tar_bytes.extend_from_slice(&chunk); } - extract_first_tar_entry(&tar_bytes).map_err(|err| { - Error::config(format!( - "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", - )) - }) + if extract_single_file { + extract_first_tar_entry(&tar_bytes).map_err(|err| { + Error::config(format!( + "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", + )) + }) + } else { + Ok(tar_bytes) + } } fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index b52cb87836..500dcfbd66 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -112,6 +112,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig { stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: "info".to_string(), supervisor_bin: PathBuf::from("/tmp/openshell-sandbox"), + supervisor_runtime: PathBuf::from("/tmp/openshell-runtime"), guest_tls: Some(DockerGuestTlsPaths { ca: PathBuf::from("/tmp/ca.crt"), cert: PathBuf::from("/tmp/tls.crt"), @@ -1068,6 +1069,17 @@ fn container_create_body_sets_driver_owned_pids_limit() { assert_eq!(host_config.pids_limit, Some(DEFAULT_SANDBOX_PIDS_LIMIT)); } +#[test] +fn admitted_container_does_not_restart() { + let body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); + let restart_policy = body + .host_config + .expect("host config") + .restart_policy + .expect("restart policy"); + assert_eq!(restart_policy.name, Some(RestartPolicyNameEnum::NO)); +} + #[test] fn build_environment_sets_docker_tls_paths() { let env = build_environment(&test_sandbox(), &runtime_config()); @@ -1417,14 +1429,15 @@ fn build_binds_uses_docker_tls_directory() { .filter_map(|bind| bind.split(':').nth(1).map(String::from)) .collect::>(); assert!(targets.contains(&SUPERVISOR_MOUNT_PATH.to_string())); + assert!(targets.contains(&SUPERVISOR_RUNTIME_MOUNT_PATH.to_string())); assert!(targets.contains(&TLS_CA_MOUNT_PATH.to_string())); assert!(targets.contains(&TLS_CERT_MOUNT_PATH.to_string())); assert!(targets.contains(&TLS_KEY_MOUNT_PATH.to_string())); - assert!( - targets - .iter() - .all(|target| target.starts_with(TLS_MOUNT_DIR) || target == SUPERVISOR_MOUNT_PATH) - ); + assert!(targets.iter().all(|target| { + target.starts_with(TLS_MOUNT_DIR) + || target == SUPERVISOR_MOUNT_PATH + || target == SUPERVISOR_RUNTIME_MOUNT_PATH + })); } #[test] diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 714b7d05c9..ab2a9015dd 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } +openshell-isolation = { path = "../openshell-isolation" } openshell-policy = { path = "../openshell-policy" } tokio = { workspace = true } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 84d7029de4..b0ebdfa660 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -284,7 +284,10 @@ const KUBERNETES_DRIVER_RESERVED_VOLUME_NAMES: &[&str] = &[ WORKSPACE_VOLUME_NAME, ]; -const KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS: &[&str] = &[SERVICE_ACCOUNT_TOKEN_MOUNT_PATH]; +const KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS: &[&str] = &[ + SERVICE_ACCOUNT_TOKEN_MOUNT_PATH, + openshell_core::driver_utils::SUPERVISOR_CONTAINER_DIR, +]; fn validate_kubernetes_driver_volumes( volumes: &[KubernetesDriverVolumeConfig], @@ -2335,6 +2338,7 @@ fn extract_image_size(message: &str) -> Option { /// Path where the supervisor binary is mounted inside the agent container. const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_DIR; +const IN_POD_ISOLATION_BACKEND_NAME: &str = "in-pod"; /// Name of the volume used to side-load the supervisor binary. const SUPERVISOR_VOLUME_NAME: &str = "openshell-supervisor-bin"; @@ -2368,7 +2372,7 @@ const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; const SIDECAR_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_CLIENT_TLS_DIR; -/// Build the emptyDir volume that holds the supervisor binary. +/// Build the emptyDir volume that holds the trusted supervisor runtime. /// /// The init container writes the binary here; the agent container reads it. fn supervisor_volume() -> serde_json::Value { @@ -2378,7 +2382,7 @@ fn supervisor_volume() -> serde_json::Value { }) } -/// Build the read-only volume mount for the supervisor binary in the agent container. +/// Build the read-only volume mount for the trusted supervisor runtime. fn supervisor_volume_mount() -> serde_json::Value { serde_json::json!({ "name": SUPERVISOR_VOLUME_NAME, @@ -2390,8 +2394,8 @@ fn supervisor_volume_mount() -> serde_json::Value { /// Build an image volume that mounts the supervisor OCI image directly. /// /// Requires Kubernetes >= v1.33 (`ImageVolume` beta) or >= v1.36 (GA). -/// The entire image filesystem is mounted read-only, making the binary -/// available at `{SUPERVISOR_MOUNT_PATH}/openshell-sandbox`. +/// The entire image filesystem is mounted read-only, making the supervisor and +/// its network-setup helpers available from one driver-controlled artifact. fn supervisor_image_volume( supervisor_image: &str, supervisor_image_pull_policy: &str, @@ -2408,26 +2412,23 @@ fn supervisor_image_volume( }) } -/// Build the init container that copies the supervisor binary into the emptyDir. +/// Build the init container that copies the trusted supervisor runtime into the emptyDir. /// /// The supervisor image contains the supervisor binary at `/openshell-sandbox`. -/// We invoke that binary with the `copy-self` subcommand so it copies itself -/// into the shared emptyDir volume, where the agent container then executes it -/// from a fixed, writable path. This pattern (binary self-copy) avoids requiring -/// `sh`/`cp` in the supervisor image and mirrors the approach used by argoexec's -/// emissary executor. +/// We invoke the supervisor's built-in installer so the binary, network helper +/// binaries, and their libraries all come from the supervisor image. The agent +/// container mounts the resulting volume read-only. fn supervisor_init_container( supervisor_image: &str, supervisor_image_pull_policy: &str, ) -> serde_json::Value { - let installed_path = format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"); let mut spec = serde_json::json!({ "name": SUPERVISOR_INIT_CONTAINER_NAME, "image": supervisor_image, "command": [ SUPERVISOR_IMAGE_BINARY_PATH, - "copy-self", - installed_path, + "copy-runtime", + SUPERVISOR_MOUNT_PATH, ], "securityContext": {"runAsUser": 0}, "volumeMounts": [{ @@ -2644,6 +2645,67 @@ fn has_upstream_proxy_credentials(params: &SandboxPodParams<'_>) -> bool { params.proxy_auth_secret_name.is_some() && params.proxy_auth_secret_key.is_some() } +fn apply_topology_descriptor(pod_template: &mut serde_json::Value) -> bool { + let Some(containers) = pod_template + .pointer_mut("/spec/containers") + .and_then(serde_json::Value::as_array_mut) + else { + return false; + }; + let index = containers + .iter() + .position(|container| container.get("name").and_then(|name| name.as_str()) == Some("agent")) + .unwrap_or(0); + let Some(command) = containers + .get_mut(index) + .and_then(|container| container.get_mut("command")) + .and_then(serde_json::Value::as_array_mut) + else { + return false; + }; + remove_protected_topology_arguments(command); + command.extend([ + serde_json::json!(format!( + "--topology-backend-name={IN_POD_ISOLATION_BACKEND_NAME}" + )), + serde_json::json!(format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + )), + serde_json::json!("--topology-payload-base64="), + ]); + true +} + +fn remove_protected_topology_arguments(arguments: &mut Vec) { + const PROTECTED: &[&str] = &[ + "--isolation-backend", + "--topology-backend-name", + "--topology-version", + "--topology-payload-base64", + ]; + let mut remove_value = false; + arguments.retain(|argument| { + if remove_value { + remove_value = false; + return false; + } + let Some(argument) = argument.as_str() else { + return true; + }; + for protected in PROTECTED { + if argument == *protected { + remove_value = true; + return false; + } + if argument.starts_with(&format!("{protected}=")) { + return false; + } + } + true + }); +} + fn sidecar_state_volume_mount() -> serde_json::Value { serde_json::json!({ "name": SIDECAR_STATE_VOLUME_NAME, @@ -3341,7 +3403,7 @@ fn sandbox_to_k8s_spec( &driver_config, inject_workspace, params, - ), + )?, ); if !template.agent_socket_path.is_empty() { root.insert( @@ -3375,10 +3437,48 @@ fn sandbox_to_k8s_spec( &driver_config, inject_workspace, params, - ), + )?, ); } + if params.topology == SupervisorTopology::Combined { + let selected = root + .get("podTemplate") + .and_then(|template| template.pointer("/spec/containers")) + .and_then(serde_json::Value::as_array) + .and_then(|containers| { + containers.iter().find(|container| { + container.get("name").and_then(|name| name.as_str()) == Some("agent") + }) + }) + .and_then(|container| container.get("command")) + .and_then(serde_json::Value::as_array) + .is_some_and(|command| { + [ + "--topology-backend-name=in-pod".to_string(), + format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ), + "--topology-payload-base64=".to_string(), + ] + .iter() + .all(|expected| { + command + .iter() + .filter(|argument| argument.as_str() == Some(expected.as_str())) + .count() + == 1 + }) + }); + if !selected { + return Err( + "failed to apply the admitted in-pod isolation backend to the supervisor command" + .to_string(), + ); + } + } + Ok(serde_json::Value::Object( std::iter::once(("spec".to_string(), serde_json::Value::Object(root))).collect(), )) @@ -3404,6 +3504,7 @@ fn sandbox_template_to_k8s( inject_workspace, params, ) + .expect("test pod template should accept the selected isolation backend") } #[cfg(test)] @@ -3425,6 +3526,7 @@ fn sandbox_template_to_k8s_with_gpu_requirements( inject_workspace, params, ) + .expect("test pod template should accept the selected isolation backend") } fn sandbox_template_to_k8s_with_validated_config( @@ -3435,7 +3537,7 @@ fn sandbox_template_to_k8s_with_validated_config( driver_config: &KubernetesSandboxDriverConfig, inject_workspace: bool, params: &SandboxPodParams<'_>, -) -> serde_json::Value { +) -> Result { let mut metadata = serde_json::Map::new(); let mut pod_labels = template .labels @@ -3746,7 +3848,16 @@ fn sandbox_template_to_k8s_with_validated_config( match params.topology { SupervisorTopology::Combined => { + // A bound topology is one-shot. A new supervisor must receive a + // newly admitted descriptor instead of restarting an old binding. + result["spec"]["restartPolicy"] = serde_json::json!("Never"); apply_supervisor_sideload_with_params(&mut result, params); + if !apply_topology_descriptor(&mut result) { + return Err( + "failed to apply the admitted in-pod isolation backend to the supervisor command" + .to_string(), + ); + } } SupervisorTopology::Sidecar => { apply_supervisor_sidecar_topology( @@ -3770,7 +3881,7 @@ fn sandbox_template_to_k8s_with_validated_config( ); } - result + Ok(result) } fn apply_pod_driver_config( @@ -5125,6 +5236,42 @@ mod tests { assert!(err.contains("/var/run/secrets/openshell")); } + #[test] + fn driver_config_reserves_the_complete_supervisor_runtime_mount() { + for mount_path in [ + "/opt/openshell", + SUPERVISOR_MOUNT_PATH, + "/opt/openshell/bin/openshell-runtime/lib", + ] { + let spec = SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": mount_path + }] + } + } + }))), + ..SandboxTemplate::default() + }), + ..SandboxSpec::default() + }; + + let err = kubernetes_driver_config_for_spec(Some(&spec), None).unwrap_err(); + assert!( + err.contains("reserved OpenShell path"), + "expected {mount_path:?} to conflict with the supervisor runtime: {err}" + ); + } + } + #[test] fn driver_config_allows_spiffe_workload_path_without_provider_spiffe() { let spec = SandboxSpec { @@ -5447,6 +5594,72 @@ mod tests { ); } + #[test] + fn trusted_driver_selects_in_pod_isolation_backend() { + let mut pod_template = serde_json::json!({ + "spec": { "containers": [{ + "name": "agent", + "command": [ + "/openshell/bin/openshell-sandbox", + "--isolation-backend=legacy", + "--topology-version=999" + ], + "args": [ + "/bin/tool", + "--isolation-backend=legacy", + "--topology-backend-name", "attacker", + "--topology-payload-base64=Zm9yZ2Vk" + ] + }] } + }); + + assert!(apply_topology_descriptor(&mut pod_template)); + + assert_eq!( + pod_template["spec"]["containers"][0]["command"], + serde_json::json!([ + "/openshell/bin/openshell-sandbox", + "--topology-backend-name=in-pod", + format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ), + "--topology-payload-base64=", + "--" + ]) + ); + assert_eq!( + pod_template["spec"]["containers"][0]["args"], + serde_json::json!([ + "/bin/tool", + "--isolation-backend=legacy", + "--topology-backend-name", + "attacker", + "--topology-payload-base64=Zm9yZ2Vk" + ]), + "arguments after the trusted delimiter belong to the workload" + ); + } + + #[test] + fn admitted_pod_does_not_restart() { + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate::default(), + false, + &std::collections::HashMap::new(), + true, + &SandboxPodParams::default(), + ); + + assert_eq!(pod_template["spec"]["restartPolicy"], "Never"); + } + + #[test] + fn in_pod_selection_fails_when_no_agent_command_exists() { + let mut pod_template = serde_json::json!({ "spec": { "containers": [] } }); + assert!(!apply_topology_descriptor(&mut pod_template)); + } + #[test] fn supervisor_sideload_replaces_spoofed_identity_environment() { let mut pod_template = serde_json::json!({ @@ -5568,17 +5781,19 @@ mod tests { assert_eq!(init_containers[0]["imagePullPolicy"], "IfNotPresent"); // The init container must invoke the binary directly with - // `copy-self ` rather than depending on shell utilities. + // `copy-runtime ` rather than depending on shell utilities or + // helpers from the workload image. let init_command = init_containers[0]["command"] .as_array() .expect("init container command should be set"); - assert_eq!(init_command.len(), 3, "expected [binary, copy-self, dest]"); - assert_eq!(init_command[0], SUPERVISOR_IMAGE_BINARY_PATH); - assert_eq!(init_command[1], "copy-self"); assert_eq!( - init_command[2].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") + init_command.len(), + 3, + "expected [binary, copy-runtime, dest]" ); + assert_eq!(init_command[0], SUPERVISOR_IMAGE_BINARY_PATH); + assert_eq!(init_command[1], "copy-runtime"); + assert_eq!(init_command[2].as_str().unwrap(), SUPERVISOR_MOUNT_PATH); assert!( !init_command.iter().any(|v| v == "sh"), "init container must not depend on a shell" diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index 8b3e014e8c..6146b07ba6 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-isolation = { path = "../openshell-isolation" } openshell-otel = { path = "../openshell-otel" } tokio = { workspace = true } diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index abb9d69dd2..e5eb22d49f 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -468,6 +468,18 @@ fn upstream_proxy_cli_args(config: &PodmanComputeConfig) -> Vec { args } +fn in_pod_topology_descriptor_args() -> Vec { + vec![ + "--topology-backend-name=in-pod".to_string(), + format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ), + "--topology-payload-base64=".to_string(), + "--".to_string(), + ] +} + fn build_env( sandbox: &DriverSandbox, config: &PodmanComputeConfig, @@ -1072,6 +1084,7 @@ pub fn build_container_spec_for_image( driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), ]; command.extend(upstream_proxy_cli_args(config)); + command.extend(in_pod_topology_descriptor_args()); let container_spec = ContainerSpec { name, @@ -1094,9 +1107,9 @@ pub fn build_container_spec_for_image( // the supervisor path as an argument instead of executing it directly. entrypoint: vec![SUPERVISOR_BINARY_PATH.into()], // Keep Podman's existing /sandbox workspace contract explicit while - // the supervisor supports driver-selected workdirs. Operator-owned - // corporate proxy flags follow it; the workload command comes from - // the reserved environment variable. + // the supervisor supports driver-selected workdirs. Trusted operator + // proxy and topology arguments follow it; workload argv remains in + // the reserved environment transport. command, // Force the supervisor to run as root (UID 0). Sandbox images may // set a non-root USER directive (e.g. `USER sandbox`), but the @@ -2098,6 +2111,20 @@ mod tests { .collect() } + #[test] + fn container_spec_admits_the_in_pod_backend_by_default() { + let spec = build_container_spec(&test_sandbox("test-id", "test-name"), &test_config()); + let command = spec_command(&spec); + + assert!(command.contains(&"--topology-backend-name=in-pod".to_string())); + assert!(command.contains(&format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ))); + assert!(command.contains(&"--topology-payload-base64=".to_string())); + assert_eq!(command.last().map(String::as_str), Some("--")); + } + #[test] fn container_spec_passes_operator_proxy_on_supervisor_argv() { let sandbox = test_sandbox("test-id", "test-name"); diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index ebcb9d2bc2..4a8c4d0425 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -20,6 +20,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } +openshell-isolation = { path = "../openshell-isolation" } openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } openshell-vfio = { path = "../openshell-vfio" } diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 19ac66c3f9..05b9933459 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -10,7 +10,7 @@ Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) fo flowchart LR subgraph host["Host process"] gateway["openshell-server
(compute::vm::spawn)"] - driver["openshell-driver-vm
├── libkrun (VM)
├── gvproxy (net)
└── openshell-sandbox.zst"] + driver["openshell-driver-vm
├── libkrun (VM)
├── gvproxy (net)
├── openshell-sandbox.zst
└── openshell-runtime.tar.zst"] gateway <-->|"gRPC over UDS
compute-driver.sock"| driver end @@ -35,7 +35,7 @@ Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside th mise run gateway:vm ``` -First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the bundled guest supervisor. Subsequent runs are cached. +First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the bundled guest supervisor plus its trusted network-helper runtime. The latter uses Docker Buildx to materialize the same runtime shipped in the supervisor image. Subsequent runs are cached. By default `mise run gateway:vm`: @@ -94,7 +94,7 @@ If you want to drive the launch yourself instead of using `mise run gateway:vm` ```shell # 1. Stage runtime artifacts + supervisor bundle into target/vm-runtime-compressed/ mise run vm:setup -mise run vm:supervisor # if openshell-sandbox.zst is not already present +mise run vm:supervisor # builds openshell-sandbox.zst and its trusted helper runtime # 2. Build both binaries with the staged artifacts embedded OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ diff --git a/crates/openshell-driver-vm/build.rs b/crates/openshell-driver-vm/build.rs index 1763590545..17f1f00bf8 100644 --- a/crates/openshell-driver-vm/build.rs +++ b/crates/openshell-driver-vm/build.rs @@ -21,6 +21,7 @@ fn main() { "libkrunfw.5.dylib.zst", "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ] { println!("cargo:rerun-if-changed={dir}/{name}"); @@ -38,7 +39,13 @@ fn main() { println!("cargo:warning=VM runtime not available for {target_os}-{target_arch}"); generate_stub_resources( &out_dir, - &["libkrun", "libkrunfw", "openshell-sandbox.zst", "umoci.zst"], + &[ + "libkrun", + "libkrunfw", + "openshell-sandbox.zst", + "openshell-runtime.tar.zst", + "umoci.zst", + ], ); return; } @@ -56,6 +63,7 @@ fn main() { &format!("{libkrunfw_name}.zst"), "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ], ); @@ -75,6 +83,7 @@ fn main() { &format!("{libkrunfw_name}.zst"), "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ], ); @@ -92,6 +101,10 @@ fn main() { "openshell-sandbox.zst".to_string(), "openshell-sandbox.zst".to_string(), ), + ( + "openshell-runtime.tar.zst".to_string(), + "openshell-runtime.tar.zst".to_string(), + ), ("umoci.zst".to_string(), "umoci.zst".to_string()), ]; @@ -135,6 +148,7 @@ fn main() { &format!("{libkrunfw_name}.zst"), "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ], ); diff --git a/crates/openshell-driver-vm/runtime/README.md b/crates/openshell-driver-vm/runtime/README.md index 11aab67f43..13bd5cac62 100644 --- a/crates/openshell-driver-vm/runtime/README.md +++ b/crates/openshell-driver-vm/runtime/README.md @@ -36,7 +36,7 @@ VM sandboxes can run the same supervisor enforcement path as other backends. # Download the current pre-built runtime and stage compressed artifacts mise run vm:setup -# Build the bundled guest supervisor +# Build the bundled guest supervisor and trusted helper runtime (requires Docker Buildx) mise run vm:supervisor # Build the gateway and VM driver with embedded runtime artifacts diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 14dbc0466b..f411972d6a 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -117,6 +117,10 @@ ensure_target_runtime() { cp /opt/openshell/bin/openshell-sandbox "$image_root/opt/openshell/bin/openshell-sandbox" chmod 0755 "$image_root/opt/openshell/bin/openshell-sandbox" fi + if [ -d /opt/openshell/bin/openshell-runtime ]; then + rm -rf "$image_root/opt/openshell/bin/openshell-runtime" + cp -a /opt/openshell/bin/openshell-runtime "$image_root/opt/openshell/bin/openshell-runtime" + fi touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow" if ! grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then @@ -214,14 +218,29 @@ exec_supervisor_in_newroot() { "${bootstrap}/lib64/ld-linux-aarch64.so.1"; do if [ -x "/newroot${loader}" ]; then lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu" - exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" \ + --workdir /sandbox \ + --topology-backend-name=in-pod \ + --topology-version=@ISOLATION_INTERFACE_VERSION@ \ + --topology-payload-base64= \ + -- fi done - exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$supervisor" \ + --workdir /sandbox \ + --topology-backend-name=in-pod \ + --topology-version=@ISOLATION_INTERFACE_VERSION@ \ + --topology-payload-base64= \ + -- fi if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then - exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox --workdir /sandbox + exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox \ + --workdir /sandbox \ + --topology-backend-name=in-pod \ + --topology-version=@ISOLATION_INTERFACE_VERSION@ \ + --topology-payload-base64= \ + -- fi done @@ -837,7 +856,12 @@ ts "starting openshell-sandbox supervisor" if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then exec_supervisor_in_newroot fi -exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox +exec /opt/openshell/bin/openshell-sandbox \ + --workdir /sandbox \ + --topology-backend-name=in-pod \ + --topology-version=@ISOLATION_INTERFACE_VERSION@ \ + --topology-payload-base64= \ + -- } if [ "${1:-}" != "--post-overlay" ]; then diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 13e57f546d..05ba97b9dc 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -4453,10 +4453,12 @@ fn build_guest_environment( || guest_visible_openshell_endpoint(&config.openshell_endpoint), String::from, ); - // 1. User-supplied environment (lowest priority). + // User-supplied values travel only through the serialized child-environment + // channel. They must not become guest-init or supervisor environment + // variables: guest init runs as root and sources only driver-owned keys, + // while the supervisor applies this map when it launches workload code. let user_env = merged_environment(sandbox); let mut environment: HashMap = HashMap::new(); - environment.extend(user_env.clone()); if !user_env.is_empty() && let Ok(json) = serde_json::to_string(&user_env) { diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 9046913c9d..ced6bff9f8 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -11,10 +11,13 @@ use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; const SUPERVISOR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/openshell-sandbox.zst")); +const SUPERVISOR_RUNTIME: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/openshell-runtime.tar.zst")); const UMOCI: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/umoci.zst")); const ROOTFS_VARIANT_MARKER: &str = ".openshell-rootfs-variant"; const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh"; const SANDBOX_SUPERVISOR_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; +const SANDBOX_SUPERVISOR_RUNTIME_PATH: &str = "/opt/openshell/bin/openshell-runtime"; const SANDBOX_UMOCI_PATH: &str = openshell_core::container_paths::VM_UMOCI_PATH; const SANDBOX_OWNER_NORMALIZED_MARKER: &str = openshell_core::container_paths::VM_SANDBOX_OWNER_NORMALIZED_MARKER; @@ -362,11 +365,12 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> if let Some(parent) = init_path.parent() { fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?; } - fs::write( - &init_path, - include_str!("../scripts/openshell-vm-sandbox-init.sh"), - ) - .map_err(|e| format!("write {}: {e}", init_path.display()))?; + let init_script = include_str!("../scripts/openshell-vm-sandbox-init.sh").replace( + "@ISOLATION_INTERFACE_VERSION@", + &openshell_isolation::contract::INTERFACE_VERSION.to_string(), + ); + fs::write(&init_path, init_script) + .map_err(|e| format!("write {}: {e}", init_path.display()))?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt as _; @@ -376,6 +380,7 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> } ensure_supervisor_binary(rootfs)?; + ensure_supervisor_runtime(rootfs)?; ensure_umoci_binary(rootfs)?; let opt_dir = rootfs.join("opt/openshell"); @@ -395,6 +400,7 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> pub fn validate_sandbox_rootfs(rootfs: &Path) -> Result<(), String> { require_rootfs_path(rootfs, SANDBOX_GUEST_INIT_PATH)?; require_rootfs_path(rootfs, SANDBOX_SUPERVISOR_PATH)?; + validate_supervisor_runtime(rootfs)?; require_rootfs_path(rootfs, SANDBOX_UMOCI_PATH)?; require_any_rootfs_path(rootfs, &["/bin/bash"])?; require_any_rootfs_path(rootfs, &["/bin/mount", "/usr/bin/mount"])?; @@ -871,6 +877,78 @@ fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> { Ok(()) } +fn ensure_supervisor_runtime(rootfs: &Path) -> Result<(), String> { + if validate_supervisor_runtime(rootfs).is_ok() { + return Ok(()); + } + if SUPERVISOR_RUNTIME.is_empty() { + return Err( + "trusted supervisor helper runtime not embedded. Build openshell-driver-vm with OPENSHELL_VM_RUNTIME_COMPRESSED_DIR set and run `mise run vm:supervisor` first" + .to_string(), + ); + } + + install_supervisor_runtime_archive(rootfs, SUPERVISOR_RUNTIME) +} + +fn install_supervisor_runtime_archive(rootfs: &Path, archive_bytes: &[u8]) -> Result<(), String> { + let destination = rootfs.join("opt/openshell/bin"); + fs::create_dir_all(&destination) + .map_err(|e| format!("create {}: {e}", destination.display()))?; + let decoder = zstd::Decoder::new(Cursor::new(archive_bytes)) + .map_err(|e| format!("decompress supervisor runtime: {e}"))?; + let mut archive = tar::Archive::new(decoder); + for entry in archive + .entries() + .map_err(|e| format!("open supervisor runtime archive: {e}"))? + { + let mut entry = entry.map_err(|e| format!("read supervisor runtime archive: {e}"))?; + let kind = entry.header().entry_type(); + if !kind.is_file() && !kind.is_dir() { + return Err( + "supervisor runtime archive contains a non-materialized link or special file" + .to_string(), + ); + } + if !entry + .unpack_in(&destination) + .map_err(|e| format!("extract supervisor runtime archive: {e}"))? + { + return Err("supervisor runtime archive contains a path outside its root".to_string()); + } + } + validate_supervisor_runtime(rootfs) +} + +fn validate_supervisor_runtime(rootfs: &Path) -> Result<(), String> { + let runtime = rootfs.join(SANDBOX_SUPERVISOR_RUNTIME_PATH.trim_start_matches('/')); + let has_ip = ["sbin/ip", "usr/sbin/ip", "bin/ip", "usr/bin/ip"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_nft = ["sbin/nft", "usr/sbin/nft", "usr/bin/nft"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_loader = fs::read_dir(runtime.join("lib")) + .ok() + .into_iter() + .flatten() + .filter_map(Result::ok) + .any(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("ld-musl-") && name.ends_with(".so.1")) + }); + if has_ip && has_nft && has_loader { + Ok(()) + } else { + Err(format!( + "trusted supervisor helper runtime '{}' is incomplete", + runtime.display() + )) + } +} + fn ensure_umoci_binary(rootfs: &Path) -> Result<(), String> { let path = rootfs.join(SANDBOX_UMOCI_PATH.trim_start_matches('/')); if UMOCI.is_empty() { @@ -979,6 +1057,19 @@ mod tests { assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file()); assert!(rootfs.join("opt/openshell/bin/umoci").is_file()); + assert!( + rootfs + .join("opt/openshell/bin/openshell-runtime/usr/sbin/nft") + .is_file() + ); + let init_script = fs::read_to_string(rootfs.join("srv/openshell-vm-sandbox-init.sh")) + .expect("read guest init"); + assert!(init_script.contains("--topology-backend-name=in-pod")); + assert!(init_script.contains(&format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ))); + assert!(!init_script.contains("@ISOLATION_INTERFACE_VERSION@")); assert!(rootfs.join("sandbox").is_dir()); assert!(rootfs.join("image-cache").is_dir()); assert!(rootfs.join("lower").is_dir()); @@ -1010,6 +1101,44 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn supervisor_runtime_archive_materializes_below_the_trusted_path() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + let mut tar_bytes = Vec::new(); + { + let mut archive = tar::Builder::new(&mut tar_bytes); + for (path, bytes, mode) in [ + ("openshell-runtime/usr/sbin/ip", b"ip".as_slice(), 0o755), + ("openshell-runtime/usr/sbin/nft", b"nft".as_slice(), 0o755), + ( + "openshell-runtime/lib/ld-musl-test.so.1", + b"loader".as_slice(), + 0o755, + ), + ] { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(mode); + header.set_entry_type(tar::EntryType::Regular); + header.set_cksum(); + archive + .append_data(&mut header, path, bytes) + .expect("append runtime entry"); + } + archive.finish().expect("finish runtime archive"); + } + let compressed = zstd::encode_all(Cursor::new(tar_bytes), 1).expect("compress runtime"); + + install_supervisor_runtime_archive(&rootfs, &compressed).expect("install runtime"); + validate_supervisor_runtime(&rootfs).expect("validate runtime"); + assert!( + rootfs + .join("opt/openshell/bin/openshell-runtime/usr/sbin/nft") + .is_file() + ); + } + #[test] fn prepare_sandbox_rootfs_preserves_image_workdir_contents_in_rootfs() { let dir = unique_temp_dir(); @@ -1230,6 +1359,13 @@ mod tests { } fn write_fake_runtime_binaries(rootfs: &Path) { + let helper_runtime = rootfs.join("opt/openshell/bin/openshell-runtime"); + fs::create_dir_all(helper_runtime.join("usr/sbin")).expect("create helper bin directory"); + fs::create_dir_all(helper_runtime.join("lib")).expect("create helper lib directory"); + fs::write(helper_runtime.join("usr/sbin/ip"), b"ip").expect("write ip helper"); + fs::write(helper_runtime.join("usr/sbin/nft"), b"nft").expect("write nft helper"); + fs::write(helper_runtime.join("lib/ld-musl-test.so.1"), b"loader") + .expect("write helper loader"); fs::write( rootfs.join("opt/openshell/bin/openshell-sandbox"), b"sandbox", diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index c653db84dd..5f12245ee8 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-extension-core = { path = "../openshell-extension-core" } +openshell-isolation = { path = "../openshell-isolation" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } @@ -25,6 +26,7 @@ openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-mid openshell-supervisor-process = { path = "../openshell-supervisor-process" } # Async runtime +async-trait = "0.1" tokio = { workspace = true } # gRPC (tonic::Status downcast in error mapping) @@ -38,12 +40,13 @@ clap = { workspace = true } miette = { workspace = true } # Unix ownership for Kubernetes sidecar init setup -nix = { workspace = true } +nix = { workspace = true, features = ["socket"] } # TLS crypto provider install (main.rs) rustls = { workspace = true } # Serialization (serde_json::json! for OCSF unmapped fields) +base64 = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } prost = { workspace = true } diff --git a/crates/openshell-sandbox/src/inpod.rs b/crates/openshell-sandbox/src/inpod.rs new file mode 100644 index 0000000000..ede172f6b5 --- /dev/null +++ b/crates/openshell-sandbox/src/inpod.rs @@ -0,0 +1,1622 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The in-pod isolation backend (RFC 0012 runtime-selectable contract). +//! +//! This is the co-located placement: the supervisor process hosts the +//! supervisor role, the mediation service, and the backend in the agent's +//! container. It implements the object-safe boxed state chain +//! (`attach -> Bound -> confirm -> Ready -> start_agent -> Running`) over the +//! existing supervisor primitives without changing their behavior: +//! `create_netns_for_proxy` (network), supervisor-owned proxy mediation, +//! the pre-exec ceiling in `spawn_workload` (filesystem/Landlock + +//! syscall/seccomp), and procfs (binary identity). +//! +//! `attach` validates the (empty) in-pod payload and atomically binds the +//! trusted [`SandboxContext`] to the boundary it establishes: the workload +//! network namespace and backend-owned connection source come up inside +//! `attach`. The supervisor connects that source to mediation before `confirm`, so +//! `Bound` means what the RFC says it means — descriptor and context bound to +//! the same resource, mediation source available, no untrusted workload code +//! running. Each transition consumes the prior state by value, so the call +//! order, and thus "no untrusted instruction before the boundary is ready", is +//! enforced by construction. +//! +//! Execution-domain note: the in-pod backend relies on container-runtime +//! inheritance — the supervisor and every child it spawns run in the pod's +//! cgroup with the device set the CRI granted the container — so every workload +//! descendant remains in the compute driver's provisioned execution +//! environment by construction. + +use std::collections::HashMap; +#[cfg(target_os = "linux")] +use std::future::Future; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + +use async_trait::async_trait; + +use openshell_core::activity::ActivitySender; +use openshell_core::denial::DenialEvent; +use openshell_core::policy::NetworkMode; +use openshell_core::proposals::AgentProposals; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation::contract::{ + BackendError, BoundBoundary, BoundaryExec, BoundaryExitStatus, BoundaryPortForward, + BoundaryProcess, BoundarySignal, INTERFACE_VERSION, IsolationBackend, MediatedConnection, + NetworkMediationSource, ReadyBoundary, RunningBoundary, SandboxContext, + VerifiedTopologyDescriptor, +}; +use openshell_supervisor_network::identity_source::ProcfsIdentityResolver; +use openshell_supervisor_process::process::ProcessEnforcementMode; +use openshell_supervisor_process::process::ResolvedProcessIdentity; +use openshell_supervisor_process::process::ResolvedWorkspace; +use openshell_supervisor_process::run::{AgentSignaler, SpawnedAgent, spawn_workload}; +use tokio::sync::mpsc::UnboundedSender; + +#[cfg(target_os = "linux")] +use openshell_supervisor_process::netns::{NetworkNamespace, create_conformant_netns_for_proxy}; + +/// Stable name of the co-located backend implementation. +pub const IN_POD_BACKEND_NAME: &str = "in-pod"; + +// ============================================================================ +// Config and backend +// ============================================================================ + +/// Runtime collaborators the in-pod lifecycle calls need, captured once when the +/// backend is built. Move-once values (the event senders) are held behind a +/// `Mutex>` so the `&self` backend/state methods can take them exactly +/// when the matching transition fires. Policy, workload, and sandbox identity +/// are *not* here; they arrive in the trusted [`SandboxContext`] at `attach`. +pub struct InPodConfig { + /// Require the supervisor to own the execution environment's PID namespace. + pub require_exclusive_pid_namespace: bool, + pub network_enabled: bool, + pub process_enabled: bool, + pub entrypoint_pid: Arc, + pub provider_credentials: ProviderCredentialState, + /// Child environment for the agent, resolved at startup. Mutated in place by + /// `attach` if the GCE metadata loopback server fails to come up. + pub provider_env: Mutex>, + /// Process launch-time enforcement level (full privileged setup vs. + /// network-sidecar reduced mode), resolved by the supervisor at startup. + pub process_enforcement_mode: ProcessEnforcementMode, + pub resolved_process_identity: ResolvedProcessIdentity, + /// Workspace resolution already normalized against the active driver. + pub workspace: ResolvedWorkspace, + pub agent_proposals: AgentProposals, + pub openshell_endpoint: Option, + pub ssh_socket_path: Option, + /// Bypass-monitor denial / activity senders (consumed by `start_agent`). + #[cfg(target_os = "linux")] + pub bypass_denial_tx: Mutex>>, + #[cfg(target_os = "linux")] + pub bypass_activity_tx: Mutex>, + /// Co-located coordination set by supervisor-owned mediation after it has + /// connected the source and before `confirm`. + pub mediation_ready: Arc, + pub ca_file_paths: Arc>>, + pub proxy_bind_ip: Arc>>, +} + +/// The backend for the in-pod backend. Holds the per-sandbox [`InPodConfig`] and +/// hands it to the boundary on the single `attach`. +pub struct InPodBackend { + config: Mutex>, + /// Whether a prior `attach` consumed the config and then failed during + /// establishment. The one-shot event senders are consumed with it, so the + /// in-pod resource cannot be re-attached; this keeps the error truthful + /// ("attempt failed", not "already bound"). + attach_failed: AtomicBool, +} + +impl InPodBackend { + /// Build the backend from its per-sandbox runtime collaborators. + #[must_use] + pub fn new(config: InPodConfig) -> Self { + Self { + config: Mutex::new(Some(config)), + attach_failed: AtomicBool::new(false), + } + } +} + +#[async_trait] +impl IsolationBackend for InPodBackend { + fn backend_name(&self) -> &'static str { + IN_POD_BACKEND_NAME + } + + fn version(&self) -> u32 { + INTERFACE_VERSION + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + // Validate the in-pod payload: the supervisor process *is* the + // resource, so the payload carries nothing. + if !descriptor.payload().is_empty() { + return Err(BackendError::Descriptor( + "in-pod descriptor payload must be empty".to_string(), + )); + } + // `attach` never binds a resource that is already bound to an active + // boundary: the in-pod resource is this process, bindable exactly once. + let config = self + .config + .lock() + .expect("in-pod config lock") + .take() + .ok_or_else(|| { + if self.attach_failed.load(Ordering::SeqCst) { + BackendError::Attach( + "a previous in-pod attach failed during establishment; \ + the in-pod resource cannot be re-attached" + .to_string(), + ) + } else { + BackendError::Denied( + "in-pod resource is already bound to an active boundary".to_string(), + ) + } + })?; + + match establish(config, sandbox).await { + Ok(bound) => Ok(Box::new(bound)), + Err(e) => { + self.attach_failed.store(true, Ordering::SeqCst); + Err(e) + } + } + } +} + +/// Establish the in-pod boundary: standing enforcement (the workload network +/// namespace) and the mediation service (the in-pod proxy), bound atomically to +/// the trusted sandbox context. Consumes the one-shot config; failure fails +/// closed with partial state released by RAII. +async fn establish( + config: InPodConfig, + sandbox: SandboxContext, +) -> Result { + #[cfg(not(target_os = "linux"))] + return Err(BackendError::Attach( + "the in-pod RFC 0012 backend requires Linux enforcement primitives".to_string(), + )); + + if matches!(sandbox.policy.network.mode, NetworkMode::Allow) { + return Err(BackendError::Denied( + "the in-pod RFC 0012 topology does not admit unrestricted network mode; workload egress must be mediated or blocked" + .to_string(), + )); + } + + // Establish the network dimension of standing enforcement: create the + // workload's network namespace and install the bypass-detection rules. + // Filesystem and syscall are launch-time controls applied per process; + // binary identity is resolved per accepted connection. + #[cfg(target_os = "linux")] + let netns = if config.network_enabled { + create_conformant_netns_for_proxy(&sandbox.policy) + .map_err(|e| BackendError::Attach(e.to_string()))? + } else { + None + }; + + #[cfg(target_os = "linux")] + let proxy_bind_ip = netns.as_ref().map(NetworkNamespace::host_ip); + #[cfg(not(target_os = "linux"))] + let proxy_bind_ip: Option = None; + *config.proxy_bind_ip.lock().expect("proxy bind IP lock") = proxy_bind_ip; + + if config.require_exclusive_pid_namespace && std::process::id() != 1 { + return Err(BackendError::Attach( + "the in-pod topology requires the supervisor to be PID 1 in its execution environment" + .to_string(), + )); + } + let runtime = if config.require_exclusive_pid_namespace { + openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new_exclusive_pid_namespace( + ) + } else { + openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new() + }; + let network_mediation_source: Arc = if config.network_enabled + && matches!(sandbox.policy.network.mode, NetworkMode::Proxy) + { + let proxy_policy = sandbox.policy.network.proxy.as_ref().ok_or_else(|| { + BackendError::Attach("proxy mode requires a proxy configuration".to_string()) + })?; + let default_ip = proxy_bind_ip.unwrap_or_else(|| std::net::IpAddr::from([127, 0, 0, 1])); + let port = proxy_policy.http_addr.map_or(3128, |addr| addr.port()); + let listener = tokio::net::TcpListener::bind((default_ip, port)) + .await + .map_err(|error| BackendError::Attach(error.to_string()))?; + Arc::new(InPodNetworkMediationSource { + listener, + identity: ProcfsIdentityResolver { + entrypoint_pid: config.entrypoint_pid.clone(), + }, + runtime: runtime.clone(), + }) + } else { + Arc::new(InactiveNetworkMediationSource { + runtime: runtime.clone(), + }) + }; + + // Start the GCE metadata loopback server inside the namespace so Go's + // metadata client (which bypasses HTTP_PROXY) can reach it via direct + // TCP. Must come up before start_agent; on failure the GCE env vars are + // stripped so the SDK falls back cleanly. + #[cfg(target_os = "linux")] + if let Some(ns) = netns.as_ref() { + ensure_gce_metadata_server(&config, ns).await; + } + + Ok(InPodBound { + config, + sandbox, + #[cfg(target_os = "linux")] + netns, + network_mediation_source, + runtime, + }) +} + +// ============================================================================ +// Lifecycle states +// ============================================================================ + +/// Bound: the descriptor and trusted sandbox context are bound to this process's +/// boundary, and the mediation source is available. No untrusted workload code +/// is running. +struct InPodBound { + config: InPodConfig, + sandbox: SandboxContext, + #[cfg(target_os = "linux")] + netns: Option, + network_mediation_source: Arc, + runtime: Arc, +} + +#[async_trait] +impl BoundBoundary for InPodBound { + fn network_mediation_source(&self) -> Arc { + self.network_mediation_source.clone() + } + + async fn confirm(self: Box) -> Result, BackendError> { + if !self.config.process_enabled { + return Err(BackendError::Confirm( + "the co-located backend requires the process supervisor leaf".to_string(), + )); + } + // Structural-mediation check (fail closed). The proxy listener must be + // connected and the live default-deny ceiling must still be present + // before the backend advances its lifecycle. + if self.config.network_enabled + && matches!(self.sandbox.policy.network.mode, NetworkMode::Proxy) + { + #[cfg(target_os = "linux")] + if self.netns.is_none() { + return Err(BackendError::Confirm( + "proxy mode requires a workload network namespace; none established" + .to_string(), + )); + } + #[cfg(target_os = "linux")] + if let Some(netns) = self.netns.as_ref() { + let proxy_port = self + .sandbox + .policy + .network + .proxy + .as_ref() + .and_then(|proxy| proxy.http_addr) + .map_or(3128, |address| address.port()); + netns + .egress_ceiling_verifier() + .verify_bounded(proxy_port, std::time::Duration::from_secs(2)) + .await + .map_err(|error| BackendError::Confirm(error.to_string()))?; + } + if !self.config.mediation_ready.load(Ordering::Acquire) { + return Err(BackendError::Confirm( + "the supervisor has not connected network mediation to the boundary source" + .to_string(), + )); + } + } + + Ok(Box::new(InPodReady { + config: self.config, + sandbox: self.sandbox, + #[cfg(target_os = "linux")] + netns: self.netns, + runtime: self.runtime, + })) + } +} + +/// Ready: standing enforcement and mediation are confirmed. Only agent +/// activation is possible. +struct InPodReady { + config: InPodConfig, + sandbox: SandboxContext, + #[cfg(target_os = "linux")] + netns: Option, + runtime: Arc, +} + +#[async_trait] +impl ReadyBoundary for InPodReady { + async fn start_agent(self: Box) -> Result, BackendError> { + let this = *self; + let config = this.config; + let sandbox = this.sandbox; + let runtime = this.runtime; + #[cfg(target_os = "linux")] + let netns = this.netns; + + #[cfg(target_os = "linux")] + let enforcement_monitor = if let Some(netns) = netns.as_ref() + && matches!(sandbox.policy.network.mode, NetworkMode::Proxy) + { + let proxy_port = sandbox + .policy + .network + .proxy + .as_ref() + .and_then(|proxy| proxy.http_addr) + .map_or(3128, |address| address.port()); + Some( + start_egress_ceiling_monitor( + netns.egress_ceiling_verifier(), + proxy_port, + runtime.clone(), + config.require_exclusive_pid_namespace, + ) + .await?, + ) + } else { + None + }; + + // The in-pod backend creates the agent process itself; the launch-time + // controls (Landlock, seccomp, privilege drop) are applied inside + // `spawn_workload`'s pre-exec ceiling, before the first untrusted + // instruction. + let (agent, exec, port_forward): ( + Arc, + Arc, + Arc, + ) = { + let spec = &sandbox.agent; + let ca_file_paths = config.ca_file_paths.lock().expect("ca paths lock").clone(); + let provider_env = config + .provider_env + .lock() + .expect("provider_env lock") + .clone(); + + #[cfg(target_os = "linux")] + let bypass_denial_tx = config + .bypass_denial_tx + .lock() + .expect("bypass_denial_tx lock") + .take(); + #[cfg(target_os = "linux")] + let bypass_activity_tx = config + .bypass_activity_tx + .lock() + .expect("bypass_activity_tx lock") + .take(); + + let spawned = spawn_workload( + &spec.program, + &spec.args, + config.workspace.clone(), + spec.timeout_secs, + spec.interactive, + Some(sandbox.sandbox_id.as_str()), + config.openshell_endpoint.as_deref(), + config.ssh_socket_path.clone(), + // In-pod co-locates the SSH socket with the workload; it is not + // shared with a separate network-sidecar container. + false, + None, + &sandbox.policy, + config.resolved_process_identity, + config.process_enforcement_mode, + config.entrypoint_pid.clone(), + // No sidecar control channel awaits the in-pod entrypoint PID. + None, + None, + config.provider_credentials.clone(), + provider_env, + ca_file_paths, + config.agent_proposals.clone(), + #[cfg(target_os = "linux")] + netns.as_ref(), + #[cfg(target_os = "linux")] + bypass_denial_tx, + #[cfg(target_os = "linux")] + bypass_activity_tx, + Some(runtime.clone()), + ) + .await + .map_err(|error| { + runtime.deactivate(); + BackendError::Process(error.to_string()) + })?; + + let exec = spawned.boundary_exec(); + let port_forward = spawned.port_forward(); + ( + Arc::new(InPodAgentProcess::running(spawned)), + exec, + port_forward, + ) + }; + + Ok(Box::new(InPodRunning { + agent, + exec, + port_forward, + runtime, + #[cfg(target_os = "linux")] + _enforcement_monitor: enforcement_monitor, + #[cfg(target_os = "linux")] + _netns: netns, + })) + } +} + +/// Running: the agent is runnable behind the boundary. Exec, forwarding, wait, +/// and signal are available. The mediation source was retained by the +/// supervisor from the `Bound` state. +struct InPodRunning { + agent: Arc, + exec: Arc, + port_forward: Arc, + runtime: Arc, + #[cfg(target_os = "linux")] + _enforcement_monitor: Option, + /// Held to keep the network namespace alive for the boundary's life; + /// dropping the running state tears it down (RAII), which is the + /// backend reclaiming backend-private state — the contract defines no public + /// cleanup transition. + #[cfg(target_os = "linux")] + _netns: Option, +} + +#[cfg(target_os = "linux")] +async fn start_egress_ceiling_monitor( + verifier: openshell_supervisor_process::netns::EgressCeilingVerifier, + proxy_port: u16, + runtime: Arc, + exit_execution_environment_on_loss: bool, +) -> Result { + let verify: EnforcementCheck = Arc::new(move || { + let verifier = verifier.clone(); + Box::pin(async move { + verifier + .verify_bounded(proxy_port, std::time::Duration::from_secs(2)) + .await + .map_err(|error| error.to_string()) + }) + }); + start_enforcement_monitor( + runtime, + std::time::Duration::from_millis(250), + verify, + exit_execution_environment_on_loss, + ) + .await +} + +#[cfg(target_os = "linux")] +type EnforcementCheck = Arc< + dyn Fn() -> std::pin::Pin> + Send + 'static>> + + Send + + Sync, +>; + +#[cfg(target_os = "linux")] +async fn start_enforcement_monitor( + runtime: Arc, + period: std::time::Duration, + verify: EnforcementCheck, + exit_execution_environment_on_loss: bool, +) -> Result { + let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + if let Err(error) = verify().await { + let _ = ready_tx.send(Err(error.clone())); + if runtime.deactivate_for_enforcement_loss() { + report_enforcement_loss(&error); + exit_execution_environment(exit_execution_environment_on_loss); + } + return; + } + if ready_tx.send(Ok(())).is_err() { + runtime.deactivate(); + return; + } + let mut interval = tokio::time::interval(period); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + interval.tick().await; + while runtime.is_active() { + interval.tick().await; + if let Err(error) = verify().await { + if runtime.deactivate_for_enforcement_loss() { + report_enforcement_loss(&error); + exit_execution_environment(exit_execution_environment_on_loss); + } + break; + } + } + }); + ready_rx + .await + .map_err(|_| BackendError::Confirm("egress monitor failed to start".to_string()))? + .map_err(BackendError::Confirm)?; + Ok(EnforcementMonitorGuard { task }) +} + +#[cfg(target_os = "linux")] +fn exit_execution_environment(enabled: bool) { + if enabled { + // This backend admits only an exclusive workload PID namespace with + // the supervisor as PID 1. Exiting its init process makes the kernel + // terminate every remaining process in that execution environment, + // including descendants that changed process group or session. + std::process::exit(125); + } +} + +#[cfg(target_os = "linux")] +struct EnforcementMonitorGuard { + task: tokio::task::JoinHandle<()>, +} + +#[cfg(target_os = "linux")] +impl Drop for EnforcementMonitorGuard { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[cfg(target_os = "linux")] +fn report_enforcement_loss(error: &str) { + let message = format!( + "Isolation boundary lost its default-deny egress ceiling; terminating workloads [error:{error}]" + ); + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(crate::ocsf_ctx()) + .severity(openshell_ocsf::SeverityId::High) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Disabled, "enforcement_lost") + .message(message.clone()) + .build() + ); + openshell_ocsf::ocsf_emit!( + openshell_ocsf::DetectionFindingBuilder::new(crate::ocsf_ctx()) + .activity(openshell_ocsf::ActivityId::Open) + .action(openshell_ocsf::ActionId::Denied) + .disposition(openshell_ocsf::DispositionId::Blocked) + .severity(openshell_ocsf::SeverityId::High) + .is_alert(true) + .finding_info(openshell_ocsf::FindingInfo::new( + "isolation-egress-enforcement-lost", + "Isolation egress enforcement lost", + )) + .message(message) + .build() + ); +} + +impl Drop for InPodRunning { + fn drop(&mut self) { + self.runtime.deactivate(); + } +} + +impl RunningBoundary for InPodRunning { + fn agent(&self) -> Arc { + self.agent.clone() + } + fn exec(&self) -> Arc { + self.exec.clone() + } + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +// ============================================================================ +// Agent process handle +// ============================================================================ + +/// The agent process running inside the in-pod boundary. `wait` returns a stable +/// terminal status across repeated calls; signals go through the lock-free +/// pid-based [`AgentSignaler`] so they never contend with an in-flight `wait`. +struct InPodAgentProcess { + signaler: Option, + result: Arc>>>, + exited: Arc, + terminal: Arc, + runtime: Arc, +} + +#[derive(Clone)] +enum StableWaitError { + Process(String), + EnforcementLost, +} + +impl InPodAgentProcess { + fn running(spawned: SpawnedAgent) -> Self { + let signaler = spawned.signaler(); + let runtime = spawned.boundary_runtime(); + let result = Arc::new(Mutex::new(None)); + let exited = Arc::new(tokio::sync::Notify::new()); + let result_for_wait = result.clone(); + let exited_for_wait = exited.clone(); + let terminal = Arc::new(AtomicBool::new(false)); + let terminal_for_wait = terminal.clone(); + let runtime_for_wait = runtime.clone(); + tokio::spawn(async move { + let mut spawned = spawned; + let waited = spawned + .wait() + .await + .map_err(|error| StableWaitError::Process(error.to_string())) + .map(|process_status| { + process_status.signal().map_or_else( + || BoundaryExitStatus::Exited(process_status.code()), + BoundaryExitStatus::Signaled, + ) + }); + let waited = if runtime_for_wait.enforcement_was_lost() { + Err(StableWaitError::EnforcementLost) + } else { + waited + }; + terminal_for_wait.store(true, Ordering::Release); + if let Ok(mut slot) = result_for_wait.lock() { + *slot = Some(waited); + } + exited_for_wait.notify_waiters(); + }); + Self { + signaler: Some(signaler), + result, + exited, + terminal, + runtime, + } + } +} + +#[async_trait] +impl BoundaryProcess for InPodAgentProcess { + async fn wait(&self) -> Result { + loop { + let notified = self.exited.notified(); + let result = self + .result + .lock() + .map_err(|_| BackendError::Process("agent result lock poisoned".to_string()))? + .clone(); + if let Some(result) = result { + self.terminal.store(true, Ordering::Release); + return result.map_err(|error| match error { + StableWaitError::Process(message) => BackendError::Process(message), + StableWaitError::EnforcementLost => BackendError::Terminated( + "required isolation enforcement was lost".to_string(), + ), + }); + } + notified.await; + } + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + self.runtime.ensure_active()?; + if self.terminal.load(Ordering::Acquire) { + return Err(BackendError::Terminated("agent has exited".to_string())); + } + let Some(signaler) = self.signaler.as_ref() else { + // Network-only hold-open: no workload process to signal. + return Ok(()); + }; + let result = match signal { + BoundarySignal::Term => signaler.term(), + BoundarySignal::Kill => signaler.kill(), + BoundarySignal::Int => signaler.interrupt(), + BoundarySignal::Hup => signaler.hangup(), + }; + result.map_err(|e| BackendError::Process(e.to_string())) + } + + async fn terminate(&self) -> Result<(), BackendError> { + self.runtime.ensure_active()?; + if self.terminal.load(Ordering::Acquire) { + return Err(BackendError::Terminated("agent has exited".to_string())); + } + let Some(signaler) = self.signaler.as_ref() else { + return Ok(()); + }; + signaler + .kill() + .map_err(|e| BackendError::Process(e.to_string())) + } +} + +// ============================================================================ +/// In-pod mediation source: owns the listener and resolves trusted procfs +/// identity for each accepted TCP connection before handing it to mediation. +/// A stronger backend may use another resolution mechanism without changing +/// the mediation contract. +struct InPodNetworkMediationSource { + listener: tokio::net::TcpListener, + identity: ProcfsIdentityResolver, + runtime: Arc, +} + +#[async_trait] +impl NetworkMediationSource for InPodNetworkMediationSource { + async fn accept(&self) -> Result { + self.runtime.ensure_active()?; + let (stream, workload_addr) = self + .listener + .accept() + .await + .map_err(|error| BackendError::Unavailable(error.to_string()))?; + self.runtime.ensure_active()?; + let proxy_addr = stream + .local_addr() + .map_err(|error| BackendError::Unavailable(error.to_string()))?; + let resolver = self.identity.clone(); + let binary_identity = tokio::task::spawn_blocking(move || { + resolver.resolve_connection(workload_addr, proxy_addr) + }) + .await + .map_err(|error| BackendError::Unavailable(error.to_string()))?; + self.runtime.ensure_active()?; + Ok(MediatedConnection { + stream: Box::new(stream), + binary_identity, + }) + } +} + +/// A topology with no proxy listener has no mediated connections. Calling +/// `accept` is an orchestration error, so fail closed rather than fabricating a +/// connection. +struct InactiveNetworkMediationSource { + runtime: Arc, +} + +#[async_trait] +impl NetworkMediationSource for InactiveNetworkMediationSource { + async fn accept(&self) -> Result { + self.runtime.ensure_active()?; + Err(BackendError::Unavailable( + "network mediation is inactive for this admitted network mode".to_string(), + )) + } +} + +// ============================================================================ +// GCE metadata loopback server +// ============================================================================ + +/// Bring up the GCE metadata loopback server inside the network namespace, +/// stripping the GCE env vars from the agent's environment if it fails so the +/// Go SDK falls back cleanly. +#[cfg(target_os = "linux")] +async fn ensure_gce_metadata_server(config: &InPodConfig, ns: &NetworkNamespace) { + use std::time::Duration; + use tokio::time::timeout; + use tracing::{info, warn}; + + if !config + .provider_credentials + .snapshot() + .child_env + .contains_key("GCE_METADATA_HOST") + { + return; + } + + let ctx = + crate::google_cloud_metadata::MetadataContext::new(config.provider_credentials.clone()); + let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); + match ns + .bind_tcp_in_netns(openshell_core::google_cloud::METADATA_LOOPBACK_ADDR) + .await + { + Ok(listener) => { + tokio::spawn(crate::metadata_server::run(listener, ctx, ready_tx)); + if let Ok(Ok(addr)) = timeout(Duration::from_secs(5), ready_rx).await { + info!(addr = %addr, "GCE metadata loopback server ready"); + } else { + warn!("GCE metadata server failed to become ready, removing metadata env vars"); + strip_gce_env(config); + } + } + Err(e) => { + warn!(error = %e, "GCE metadata server bind failed, Go SDK may not discover credentials"); + strip_gce_env(config); + } + } +} + +/// Remove the GCE metadata env vars from both the agent's child env and the +/// provider credential state. +#[cfg(target_os = "linux")] +fn strip_gce_env(config: &InPodConfig) { + let mut env = config.provider_env.lock().expect("provider_env lock"); + env.remove("GCE_METADATA_HOST"); + env.remove("GCE_METADATA_IP"); + env.remove("METADATA_SERVER_DETECTION"); + drop(env); + config + .provider_credentials + .remove_env_key("GCE_METADATA_HOST"); +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + use openshell_isolation::AgentSpec; + use openshell_isolation::contract::{BackendRegistry, TopologyDescriptor}; + + /// A minimal in-pod config with networking disabled. The process leaf is + /// declared available so `confirm` can certify launch readiness, but tests + /// that use this fixture do not call `start_agent`. + fn minimal_config() -> InPodConfig { + InPodConfig { + require_exclusive_pid_namespace: false, + network_enabled: false, + process_enabled: true, + entrypoint_pid: Arc::new(AtomicU32::new(0)), + provider_credentials: ProviderCredentialState::from_environment( + 0, + HashMap::new(), + HashMap::new(), + HashMap::new(), + ), + provider_env: Mutex::new(HashMap::new()), + process_enforcement_mode: ProcessEnforcementMode::Full, + resolved_process_identity: ResolvedProcessIdentity::new( + Some(nix::unistd::geteuid().as_raw()), + Some(nix::unistd::getegid().as_raw()), + ), + workspace: ResolvedWorkspace::default(), + agent_proposals: AgentProposals::new(false), + openshell_endpoint: None, + ssh_socket_path: None, + #[cfg(target_os = "linux")] + bypass_denial_tx: Mutex::new(None), + #[cfg(target_os = "linux")] + bypass_activity_tx: Mutex::new(None), + mediation_ready: Arc::new(AtomicBool::new(true)), + ca_file_paths: Arc::new(Mutex::new(None)), + proxy_bind_ip: Arc::new(Mutex::new(None)), + } + } + + fn block_mode_policy() -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy { + mode: NetworkMode::Block, + proxy: None, + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + } + } + + fn descriptor() -> TopologyDescriptor { + TopologyDescriptor { + version: INTERFACE_VERSION, + backend_name: IN_POD_BACKEND_NAME.to_string(), + payload: Vec::new(), + } + } + + fn sandbox_context() -> SandboxContext { + SandboxContext { + sandbox_id: "test-sandbox".to_string(), + policy: block_mode_policy(), + agent: AgentSpec { + program: "true".to_string(), + args: vec![], + workdir: None, + timeout_secs: 0, + interactive: false, + }, + } + } + + // ----- Backend and registry ----- + + #[test] + fn backend_speaks_the_version() { + let backend = InPodBackend::new(minimal_config()); + assert_eq!(backend.backend_name(), IN_POD_BACKEND_NAME); + assert_eq!(backend.version(), INTERFACE_VERSION); + } + + #[test] + fn registry_selects_in_pod_backend() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + let (backend, _verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + assert_eq!(backend.backend_name(), IN_POD_BACKEND_NAME); + } + + #[test] + fn registry_rejects_duplicate_in_pod() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("first register"); + assert!( + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .is_err() + ); + } + + #[test] + fn registry_rejects_admission_mismatch() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + // The descriptor names in-pod, but admission expects a different backend. + assert!( + registry + .resolve(descriptor(), "some-other-backend") + .map(|_| ()) + .is_err() + ); + } + + // ----- Lifecycle (no root / no netns) ----- + + /// Drive the real in-pod chain attach -> Bound -> confirm -> Ready and prove + /// the retained mediation source survives the consuming transitions. + #[tokio::test] + async fn lifecycle_reaches_ready_and_retains_source() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + let (backend, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + + let bound = backend + .attach(verified, sandbox_context()) + .await + .expect("attach"); + let source = bound.network_mediation_source(); + let _ready = bound.confirm().await.expect("confirm"); + // Block mode has no connection source, so a retained accept fails + // closed after `Bound` is consumed. + assert!(source.accept().await.is_err()); + } + + #[tokio::test] + async fn live_source_accepts_stream_and_carries_fail_closed_identity() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let source = Arc::new(InPodNetworkMediationSource { + listener, + identity: ProcfsIdentityResolver { + entrypoint_pid: Arc::new(AtomicU32::new(0)), + }, + runtime: openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(), + }); + let client = tokio::spawn(async move { + let mut stream = tokio::net::TcpStream::connect(address).await.unwrap(); + stream.write_all(b"ping").await.unwrap(); + }); + let mut connection = source.accept().await.expect("accept"); + assert!(connection.binary_identity.is_err()); + let mut bytes = [0_u8; 4]; + connection.stream.read_exact(&mut bytes).await.unwrap(); + assert_eq!(&bytes, b"ping"); + client.await.unwrap(); + } + + #[tokio::test] + async fn pending_source_accept_rejects_connection_after_boundary_end() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let runtime = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let source = Arc::new(InPodNetworkMediationSource { + listener, + identity: ProcfsIdentityResolver { + entrypoint_pid: Arc::new(AtomicU32::new(0)), + }, + runtime: runtime.clone(), + }); + let pending = tokio::spawn({ + let source = source.clone(); + async move { source.accept().await } + }); + tokio::task::yield_now().await; + runtime.deactivate(); + let _client = tokio::net::TcpStream::connect(address).await.unwrap(); + assert!(matches!( + pending.await.unwrap(), + Err(BackendError::Terminated(_)) + )); + } + + #[tokio::test] + async fn mediation_source_failure_is_fail_static_without_ending_the_boundary() { + let runtime = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let source = InactiveNetworkMediationSource { + runtime: runtime.clone(), + }; + + assert!(matches!( + source.accept().await, + Err(BackendError::Unavailable(_)) + )); + runtime + .ensure_active() + .expect("network failure must leave Running active"); + assert!(matches!( + source.accept().await, + Err(BackendError::Unavailable(_)) + )); + } + + /// `attach` is atomic and never binds a resource that is already bound to an + /// active boundary: the in-pod resource binds exactly once. + #[tokio::test] + async fn second_attach_is_denied() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + + let (backend, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + let _bound = backend + .attach(verified, sandbox_context()) + .await + .expect("first attach"); + + let (_backend2, verified2) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("re-resolve"); + let err = backend + .attach(verified2, sandbox_context()) + .await + .map(|_| ()) + .expect_err("second attach must fail"); + assert!(matches!(err, BackendError::Denied(_))); + } + + /// The in-pod payload is empty by construction; a non-empty payload is a + /// descriptor error, validated by the backend at `attach`. + #[tokio::test] + async fn non_empty_payload_is_rejected() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + let bad = TopologyDescriptor { + version: INTERFACE_VERSION, + backend_name: IN_POD_BACKEND_NAME.to_string(), + payload: vec![1, 2, 3], + }; + let (backend, verified) = registry.resolve(bad, IN_POD_BACKEND_NAME).expect("resolve"); + let err = backend + .attach(verified, sandbox_context()) + .await + .map(|_| ()) + .expect_err("payload must be rejected"); + assert!(matches!(err, BackendError::Descriptor(_))); + } + + #[tokio::test] + async fn unrestricted_network_mode_is_not_admitted() { + let backend = Arc::new(InPodBackend::new(minimal_config())); + let mut registry = BackendRegistry::new(); + registry.register(backend.clone()).expect("register"); + let (_resolved, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + let mut sandbox = sandbox_context(); + sandbox.policy.network.mode = NetworkMode::Allow; + let error = backend + .attach(verified, sandbox) + .await + .map(|_| ()) + .expect_err("unrestricted egress cannot conform"); + assert_eq!( + error.kind(), + openshell_isolation::contract::BackendErrorKind::Denied + ); + } + + #[tokio::test] + async fn confirm_rejects_missing_launch_control_leaf() { + let mut config = minimal_config(); + config.process_enabled = false; + let backend = Arc::new(InPodBackend::new(config)); + let mut registry = BackendRegistry::new(); + registry.register(backend.clone()).expect("register"); + let (_resolved, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + let bound = backend + .attach(verified, sandbox_context()) + .await + .expect("attach"); + let error = bound + .confirm() + .await + .map(|_| ()) + .expect_err("Ready requires launch controls"); + assert!(matches!(error, BackendError::Confirm(_))); + } + + #[tokio::test] + async fn failed_agent_start_ends_boundary_and_invalidates_retained_source() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + let (backend, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + let mut sandbox = sandbox_context(); + sandbox.agent.program = "/definitely/missing/openshell-agent".to_string(); + let bound = backend.attach(verified, sandbox).await.expect("attach"); + let source = bound.network_mediation_source(); + let ready = bound.confirm().await.expect("confirm"); + assert!(matches!( + ready.start_agent().await, + Err(BackendError::Process(_)) + )); + assert!(matches!( + source.accept().await, + Err(BackendError::Terminated(_)) + )); + } + + #[tokio::test] + async fn normal_agent_exit_invalidates_runtime_interfaces() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + let (backend, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + let bound = backend + .attach(verified, sandbox_context()) + .await + .expect("attach"); + let source = bound.network_mediation_source(); + let ready = bound.confirm().await.expect("confirm"); + let running = ready.start_agent().await.expect("start agent"); + let agent = running.agent(); + let exec = running.exec(); + let forward = running.port_forward(); + + assert_eq!( + agent.wait().await.expect("normal exit"), + BoundaryExitStatus::Exited(0) + ); + assert!(matches!( + exec.exec(openshell_isolation::contract::ExecSpec { + program: "/bin/true".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }) + .await, + Err(BackendError::Terminated(_)) + )); + let target = openshell_isolation::contract::LoopbackTarget::new( + std::net::Ipv4Addr::LOCALHOST.into(), + 1, + ) + .expect("loopback target"); + assert!(matches!( + forward.connect(target).await, + Err(BackendError::Terminated(_)) + )); + assert!(matches!( + source.accept().await, + Err(BackendError::Terminated(_)) + )); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn enforcement_monitor_terminates_boundary_after_verification_loss() { + let runtime = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let healthy = Arc::new(AtomicBool::new(true)); + let verify: EnforcementCheck = { + let healthy = healthy.clone(); + Arc::new(move || { + let healthy = healthy.clone(); + Box::pin(async move { + if healthy.load(Ordering::Acquire) { + Ok(()) + } else { + Err("test enforcement loss".to_string()) + } + }) + }) + }; + let _monitor = start_enforcement_monitor( + runtime.clone(), + std::time::Duration::from_millis(5), + verify, + false, + ) + .await + .expect("initial enforcement verification"); + tokio::time::sleep(std::time::Duration::from_millis(15)).await; + runtime.ensure_active().expect("healthy enforcement"); + + healthy.store(false, Ordering::Release); + tokio::time::timeout(std::time::Duration::from_millis(100), async { + while runtime.is_active() { + tokio::task::yield_now().await; + } + }) + .await + .expect("monitor must terminate within its bound"); + assert!(runtime.ensure_active().is_err()); + assert!(runtime.enforcement_was_lost()); + } + + #[cfg(unix)] + #[tokio::test] + async fn enforcement_loss_kills_registered_workload_within_bound() { + use std::os::unix::process::CommandExt as _; + + let runtime = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let mut command = std::process::Command::new("/bin/sleep"); + command.arg("30").process_group(0); + let mut child = command.spawn().expect("spawn workload process"); + runtime + .register_process_group( + child.id(), + Arc::new(AtomicBool::new(false)), + Arc::new(Mutex::new(())), + ) + .expect("register workload process group"); + + let healthy = Arc::new(AtomicBool::new(true)); + let verify: EnforcementCheck = { + let healthy = healthy.clone(); + Arc::new(move || { + let healthy = healthy.clone(); + Box::pin(async move { + healthy + .load(Ordering::Acquire) + .then_some(()) + .ok_or_else(|| "test enforcement loss".to_string()) + }) + }) + }; + let _monitor = start_enforcement_monitor( + runtime.clone(), + std::time::Duration::from_millis(5), + verify, + false, + ) + .await + .expect("initial verification"); + healthy.store(false, Ordering::Release); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + match child.try_wait() { + Ok(Some(_)) => break, + Ok(None) => {} + Err(error) if error.raw_os_error() == Some(nix::libc::ECHILD) => break, + Err(error) => panic!("wait workload: {error}"), + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .expect("workload must terminate within the topology bound"); + assert!(runtime.enforcement_was_lost()); + } + + #[cfg(target_os = "linux")] + #[test] + #[ignore = "requires privileged PID namespace creation"] + #[allow(unsafe_code)] + #[allow( + clippy::zombie_processes, + reason = "PID 1 exits to make the kernel reap this deliberately unregistered descendant" + )] + fn pid_namespace_exit_helper() { + use std::io::Write as _; + use std::os::unix::process::CommandExt as _; + + let Some(ready_path) = std::env::var_os("OPENSHELL_PIDNS_TEST_READY") else { + return; + }; + let trigger_path = std::env::var_os("OPENSHELL_PIDNS_TEST_TRIGGER") + .expect("PID namespace helper trigger path"); + let identity_socket = std::env::var_os("OPENSHELL_PIDNS_TEST_SOCKET") + .expect("PID namespace helper identity socket"); + assert_eq!(std::process::id(), 1, "helper must be PID 1"); + std::os::unix::net::UnixStream::connect(&identity_socket) + .expect("connect PID 1 identity socket") + .write_all(b"pid1") + .expect("publish PID 1 identity"); + let mut command = + std::process::Command::new(std::env::current_exe().expect("current test executable")); + command + .args([ + "--ignored", + "--exact", + "inpod::tests::pid_namespace_descendant_helper", + "--nocapture", + ]) + .env("OPENSHELL_PIDNS_TEST_SOCKET", &identity_socket); + // SAFETY: `setsid` is async-signal-safe and has no captured state. + unsafe { + command.pre_exec(|| { + if nix::libc::setsid() >= 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + let _unregistered_descendant = command.spawn().expect("spawn setsid descendant"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("build monitor runtime"); + runtime.block_on(async move { + let boundary = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let healthy = Arc::new(AtomicBool::new(true)); + let verify: EnforcementCheck = { + let healthy = healthy.clone(); + Arc::new(move || { + let healthy = healthy.clone(); + Box::pin(async move { + healthy + .load(Ordering::Acquire) + .then_some(()) + .ok_or_else(|| "privileged test enforcement loss".to_string()) + }) + }) + }; + let _monitor = start_enforcement_monitor( + boundary, + std::time::Duration::from_millis(5), + verify, + true, + ) + .await + .expect("start enforcement monitor"); + std::fs::write(&ready_path, b"ready").expect("publish helper readiness"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !std::path::Path::new(&trigger_path).exists() { + assert!( + std::time::Instant::now() < deadline, + "parent did not trigger enforcement loss" + ); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + healthy.store(false, Ordering::Release); + std::future::pending::<()>().await; + }); + } + + #[cfg(target_os = "linux")] + #[test] + #[ignore = "helper for privileged PID namespace test"] + fn pid_namespace_descendant_helper() { + use std::io::Write as _; + + let Some(socket_path) = std::env::var_os("OPENSHELL_PIDNS_TEST_SOCKET") else { + return; + }; + std::os::unix::net::UnixStream::connect(socket_path) + .expect("connect descendant identity socket") + .write_all(b"descendant") + .expect("publish descendant identity"); + loop { + std::thread::park(); + } + } + + #[cfg(target_os = "linux")] + #[test] + #[ignore = "requires privileged PID namespace creation"] + fn pid_one_exit_kills_unregistered_setsid_descendant_within_bound() { + fn process_is_running(pid: u32) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + let Some((_, fields)) = stat.rsplit_once(") ") else { + return false; + }; + !matches!(fields.as_bytes().first(), Some(b'Z' | b'X')) + } + + let tempdir = tempfile::tempdir().expect("tempdir"); + let ready = tempdir.path().join("ready"); + let trigger = tempdir.path().join("trigger"); + let identity_socket = tempdir.path().join("identity.sock"); + let listener = + std::os::unix::net::UnixListener::bind(&identity_socket).expect("bind identity socket"); + listener + .set_nonblocking(true) + .expect("set identity socket nonblocking"); + let current_exe = std::env::current_exe().expect("current test executable"); + let mut namespace = std::process::Command::new("unshare") + .args(["--mount", "--pid", "--fork", "--kill-child", "--mount-proc"]) + .arg(current_exe) + .args([ + "--ignored", + "--exact", + "inpod::tests::pid_namespace_exit_helper", + "--nocapture", + ]) + .env("OPENSHELL_PIDNS_TEST_READY", &ready) + .env("OPENSHELL_PIDNS_TEST_TRIGGER", &trigger) + .env("OPENSHELL_PIDNS_TEST_SOCKET", &identity_socket) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("start isolated PID namespace"); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let mut isolated = Vec::new(); + while !ready.exists() || isolated.len() < 2 { + match listener.accept() { + Ok((stream, _)) => { + use nix::sys::socket::{getsockopt, sockopt::PeerCredentials}; + + let credentials = getsockopt(&stream, PeerCredentials) + .expect("read namespaced process credentials"); + isolated.push(u32::try_from(credentials.pid()).expect("positive peer PID")); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {} + Err(error) => panic!("accept identity connection: {error}"), + } + if let Some(status) = namespace.try_wait().expect("poll namespace") { + let stderr = namespace + .stderr + .take() + .and_then(|mut stderr| { + use std::io::Read as _; + let mut output = String::new(); + stderr.read_to_string(&mut output).ok()?; + Some(output) + }) + .unwrap_or_default(); + panic!("PID namespace helper exited early ({status}): {stderr}"); + } + assert!( + std::time::Instant::now() < deadline, + "helper readiness timeout" + ); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + + assert_eq!( + isolated.len(), + 2, + "expected PID 1 and its setsid descendant" + ); + std::fs::write(&trigger, b"exit").expect("trigger PID 1 exit"); + let status = namespace.wait().expect("wait for namespace exit"); + assert_eq!(status.code(), Some(125)); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while isolated.iter().copied().any(process_is_running) { + assert!( + std::time::Instant::now() < deadline, + "namespace descendant survived the documented termination bound" + ); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + } + + #[tokio::test] + async fn wait_reports_enforcement_loss_as_terminated() { + let process = InPodAgentProcess { + signaler: None, + result: Arc::new(Mutex::new(Some(Err(StableWaitError::EnforcementLost)))), + exited: Arc::new(tokio::sync::Notify::new()), + terminal: Arc::new(AtomicBool::new(true)), + runtime: openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(), + }; + + let error = process + .wait() + .await + .expect_err("enforcement loss is abnormal"); + assert_eq!( + error.kind(), + openshell_isolation::contract::BackendErrorKind::Terminated + ); + } + + #[tokio::test] + async fn normal_teardown_is_not_reclassified_by_inflight_verification() { + let runtime = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let calls = Arc::new(AtomicU32::new(0)); + let entered = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let verify: EnforcementCheck = { + let calls = calls.clone(); + let entered = entered.clone(); + let release = release.clone(); + Arc::new(move || { + let calls = calls.clone(); + let entered = entered.clone(); + let release = release.clone(); + Box::pin(async move { + if calls.fetch_add(1, Ordering::AcqRel) == 0 { + return Ok(()); + } + entered.notify_one(); + release.notified().await; + Err("verification completed after teardown".to_string()) + }) + }) + }; + let _monitor = start_enforcement_monitor( + runtime.clone(), + std::time::Duration::from_millis(1), + verify, + false, + ) + .await + .expect("initial verification"); + entered.notified().await; + runtime.deactivate(); + release.notify_one(); + tokio::task::yield_now().await; + + assert!(!runtime.enforcement_was_lost()); + } +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index b1c226cebd..520720deb0 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -9,6 +9,7 @@ mod activity_aggregator; mod denial_aggregator; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod google_cloud_metadata; +mod inpod; mod mechanistic_mapper; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod metadata_server; @@ -116,6 +117,7 @@ pub async fn run_sandbox( network_enabled: bool, process_enabled: bool, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + topology_descriptor: Option, ) -> Result { let (program, args) = command .split_first() @@ -145,7 +147,12 @@ pub async fn run_sandbox( } let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); - let process_enforcement_mode = process_enforcement_mode(); + let admitted_topology = topology_descriptor.is_some(); + let process_enforcement_mode = if admitted_topology { + ProcessEnforcementMode::Full + } else { + process_enforcement_mode() + }; let process_uses_sidecar_control = process_enabled && !network_enabled && sidecar_network_enforcement; let mut process_control_connection = None; @@ -390,7 +397,7 @@ pub async fn run_sandbox( // it via setns(). The RAII handle lives in this frame for the duration // of the sandbox. #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { + let netns = if network_enabled && !sidecar_network_enforcement && !admitted_topology { openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? } else { None @@ -403,6 +410,12 @@ pub async fn run_sandbox( .transpose()? .is_some_and(|snapshot| !snapshot.endpoints.is_empty()); #[cfg(target_os = "linux")] + if admitted_topology && transparent_tcp_requested { + return Err(miette::miette!( + "the RFC 0012 in-pod prototype does not yet compose its strict egress ceiling with policy DNS and transparent TCP" + )); + } + #[cfg(target_os = "linux")] let runtime_capabilities = std::env::var(openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES).ok(); #[cfg(target_os = "linux")] @@ -516,7 +529,98 @@ pub async fn run_sandbox( // API read the current value so proposals target the correct workspace. let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); - let mut networking = if network_enabled { + let mut admitted_ready: Option> = None; + let mut networking = if let Some(descriptor) = topology_descriptor { + if sidecar_network_enforcement || !network_enabled || !process_enabled { + return Err(miette::miette!( + "an admitted isolation backend requires the co-located network,process topology" + )); + } + let mediation_ready = Arc::new(AtomicBool::new(false)); + let ca_file_paths = Arc::new(std::sync::Mutex::new(None)); + let proxy_bind_ip = Arc::new(std::sync::Mutex::new(None)); + let backend = Arc::new(inpod::InPodBackend::new(inpod::InPodConfig { + require_exclusive_pid_namespace: true, + network_enabled, + process_enabled, + entrypoint_pid: entrypoint_pid.clone(), + provider_credentials: provider_credentials.clone(), + provider_env: std::sync::Mutex::new(provider_env.clone()), + process_enforcement_mode, + resolved_process_identity, + workspace: workspace.clone(), + agent_proposals: agent_proposals.clone(), + openshell_endpoint: openshell_endpoint_for_proxy.clone(), + ssh_socket_path: ssh_socket_path.clone(), + #[cfg(target_os = "linux")] + bypass_denial_tx: std::sync::Mutex::new(bypass_denial_tx.clone()), + #[cfg(target_os = "linux")] + bypass_activity_tx: std::sync::Mutex::new(bypass_activity_tx.clone()), + mediation_ready: mediation_ready.clone(), + ca_file_paths: ca_file_paths.clone(), + proxy_bind_ip: proxy_bind_ip.clone(), + })); + let mut registry = openshell_isolation::contract::BackendRegistry::new(); + registry + .register(backend) + .map_err(|error| miette::miette!(error.to_string()))?; + let (backend, verified) = registry + .resolve(descriptor, inpod::IN_POD_BACKEND_NAME) + .map_err(|error| miette::miette!(error.to_string()))?; + let bound = backend + .attach( + verified, + openshell_isolation::contract::SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + policy: policy.clone(), + agent: openshell_isolation::AgentSpec { + program: program.clone(), + args: args.to_vec(), + workdir: workdir.clone(), + timeout_secs, + interactive, + }, + }, + ) + .await + .map_err(|error| miette::miette!(error.to_string()))?; + let source = bound.network_mediation_source(); + let bind_ip = *proxy_bind_ip.lock().expect("proxy bind IP lock"); + let networking = openshell_supervisor_network::run::run_networking( + &policy, + bind_ip, + opa_engine.as_ref(), + retained_proto.as_ref(), + entrypoint_pid.clone(), + process_enabled, + &provider_credentials, + sandbox_id.as_deref(), + sandbox_name_for_agg.as_deref(), + openshell_endpoint_for_proxy.as_deref(), + inference_routes.as_deref(), + denial_tx, + activity_tx, + agent_proposals.clone(), + workspace_rx.clone(), + &upstream_proxy_args, + Some(source), + #[cfg(target_os = "linux")] + None, + ) + .await?; + ca_file_paths + .lock() + .expect("ca paths lock") + .clone_from(&networking.ca_file_paths); + mediation_ready.store(true, Ordering::Release); + admitted_ready = Some( + bound + .confirm() + .await + .map_err(|error| miette::miette!(error.to_string()))?, + ); + Some(networking) + } else if network_enabled { #[cfg(target_os = "linux")] let proxy_bind_ip = netns .as_ref() @@ -542,6 +646,7 @@ pub async fn run_sandbox( agent_proposals.clone(), workspace_rx.clone(), &upstream_proxy_args, + None, #[cfg(target_os = "linux")] transparent_runtime, ) @@ -839,36 +944,57 @@ pub async fn run_sandbox( tokio::pin!(proxy_exited); let exit_code = if process_enabled { - let ca_file_paths = networking - .as_ref() - .and_then(|n| n.ca_file_paths.clone()) - .or_else(|| { - if sidecar_network_enforcement { - sidecar_bootstrap_ca_file_paths - .clone() - .or_else(sidecar_ca_file_paths) - } else { - None + if let Some(ready) = admitted_ready.take() { + let running = ready + .start_agent() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + let agent = running.agent(); + let exit = tokio::select! { + result = agent.wait() => result.map_err(|error| miette::miette!(error.to_string()))?, + () = &mut proxy_exited => { + return Err(miette::miette!("RFC boundary mediation exited unexpectedly")); } - }); - - let (ssh_exit_tx, ssh_exit_rx) = if ssh_socket_path.is_some() { - let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - (Some(tx), Some(rx)) - } else { - (None, None) - }; - let ssh_exited: Pin + Send>> = if let Some(rx) = ssh_exit_rx { - Box::pin(async { - let _ = rx.await; - }) + }; + match exit { + openshell_isolation::contract::BoundaryExitStatus::Exited(code) => code, + openshell_isolation::contract::BoundaryExitStatus::Signaled(signal) => { + 128_i32.saturating_add(signal) + } + } } else { - Box::pin(std::future::pending()) - }; - tokio::pin!(ssh_exited); + let ca_file_paths = networking + .as_ref() + .and_then(|n| n.ca_file_paths.clone()) + .or_else(|| { + if sidecar_network_enforcement { + sidecar_bootstrap_ca_file_paths + .clone() + .or_else(sidecar_ca_file_paths) + } else { + None + } + }); - let entrypoint_started_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { + let (ssh_exit_tx, ssh_exit_rx) = if ssh_socket_path.is_some() { + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + (Some(tx), Some(rx)) + } else { + (None, None) + }; + let ssh_exited: Pin + Send>> = if let Some(rx) = ssh_exit_rx + { + Box::pin(async { + let _ = rx.await; + }) + } else { + Box::pin(std::future::pending()) + }; + tokio::pin!(ssh_exited); + + let entrypoint_started_tx = if process_uses_sidecar_control + && let Some(writer) = process_control_writer.clone() + { let (tx, rx) = tokio::sync::oneshot::channel(); tokio::spawn(async move { match rx.await { @@ -889,8 +1015,9 @@ pub async fn run_sandbox( } else { None }; - let sidecar_exit_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { + let sidecar_exit_tx = if process_uses_sidecar_control + && let Some(writer) = process_control_writer.clone() + { let exit_ack = Arc::clone(&process_exit_ack); let (tx, mut rx) = tokio::sync::mpsc::channel::< openshell_supervisor_process::run::SidecarExitReport, @@ -919,116 +1046,117 @@ pub async fn run_sandbox( None }; - let process = openshell_supervisor_process::run::run_process( - program, - args, - workspace, - timeout_secs, - interactive, - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - ssh_socket_path, - sidecar_network_enforcement, - ssh_exit_tx, - &process_policy, - resolved_process_identity, - process_enforcement_mode, - entrypoint_pid, - entrypoint_started_tx, - sidecar_exit_tx, - provider_credentials, - main_env, - ca_file_paths, - agent_proposals.clone(), - #[cfg(target_os = "linux")] - netns.as_ref(), - #[cfg(target_os = "linux")] - bypass_denial_tx, - #[cfg(target_os = "linux")] - bypass_activity_tx, - ); + let process = openshell_supervisor_process::run::run_process( + program, + args, + workspace, + timeout_secs, + interactive, + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + ssh_socket_path, + sidecar_network_enforcement, + ssh_exit_tx, + &process_policy, + resolved_process_identity, + process_enforcement_mode, + entrypoint_pid, + entrypoint_started_tx, + sidecar_exit_tx, + provider_credentials, + main_env, + ca_file_paths, + agent_proposals.clone(), + #[cfg(target_os = "linux")] + netns.as_ref(), + #[cfg(target_os = "linux")] + bypass_denial_tx, + #[cfg(target_os = "linux")] + bypass_activity_tx, + ); - if let Some(control_closed) = process_control_closed.as_mut() { - tokio::select! { - result = process => result?, - _ = control_closed => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Authoritative network-sidecar control channel closed; terminating process container" - ) - .build() - ); - return Err(miette::miette!( - "authoritative network-sidecar control channel closed" - )); - } - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - () = &mut ssh_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "SSH accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "SSH accept loop exited unexpectedly" - )); - } - } - } else { - tokio::select! { - result = process => result?, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); + if let Some(control_closed) = process_control_closed.as_mut() { + tokio::select! { + result = process => result?, + _ = control_closed => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "Authoritative network-sidecar control channel closed; terminating process container" + ) + .build() + ); + return Err(miette::miette!( + "authoritative network-sidecar control channel closed" + )); + } + () = &mut proxy_exited => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "Proxy accept loop exited unexpectedly; terminating sandbox" + ) + .build() + ); + return Err(miette::miette!( + "proxy accept loop exited unexpectedly" + )); + } + () = &mut ssh_exited => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "SSH accept loop exited unexpectedly; terminating sandbox" + ) + .build() + ); + return Err(miette::miette!( + "SSH accept loop exited unexpectedly" + )); + } } - () = &mut ssh_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "SSH accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "SSH accept loop exited unexpectedly" - )); + } else { + tokio::select! { + result = process => result?, + () = &mut proxy_exited => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "Proxy accept loop exited unexpectedly; terminating sandbox" + ) + .build() + ); + return Err(miette::miette!( + "proxy accept loop exited unexpectedly" + )); + } + () = &mut ssh_exited => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "SSH accept loop exited unexpectedly; terminating sandbox" + ) + .build() + ); + return Err(miette::miette!( + "SSH accept loop exited unexpectedly" + )); + } } } } diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 6d244fb6bc..99f69301fe 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -235,6 +235,18 @@ struct Args { /// re-signed upstream certificates and the sandbox trust bundle. #[arg(long)] upstream_proxy_ca_bundle: Option, + + /// Backend selected by the compute driver's admitted topology descriptor. + #[arg(long)] + topology_backend_name: Option, + + /// Isolation Backend interface version. + #[arg(long)] + topology_version: Option, + + /// Base64-encoded opaque backend payload. + #[arg(long)] + topology_payload_base64: Option, } /// Internal one-shot command used by the privileged supervisor to validate an @@ -683,6 +695,28 @@ fn main() -> Result<()> { proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, proxy_ca_bundle: args.upstream_proxy_ca_bundle, }; + let topology_descriptor = match ( + args.topology_backend_name, + args.topology_version, + args.topology_payload_base64, + ) { + (None, None, None) => None, + (Some(backend_name), Some(version), Some(payload)) => { + use base64::Engine as _; + Some(openshell_isolation::contract::TopologyDescriptor { + backend_name, + version, + payload: base64::engine::general_purpose::STANDARD + .decode(payload) + .into_diagnostic()?, + }) + } + _ => { + return Err(miette::miette!( + "topology descriptor requires backend name, version, and payload" + )); + } + }; run_sandbox( command, @@ -702,6 +736,7 @@ fn main() -> Result<()> { args.mode.network, args.mode.process, upstream_proxy_args, + topology_descriptor, ) .await })?; diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index dc2736a4ea..56c0703ed6 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -24,6 +24,10 @@ use openshell_core::net::{ use openshell_core::policy::ProxyPolicy; use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; use openshell_core::secrets::{self, SecretResolver, rewrite_header_line_checked}; +use openshell_isolation::contract::{ + BinaryIdentity as ContractBinaryIdentity, BoundaryDuplexStream, NetworkMediationSource, + ResolveError, +}; use openshell_ocsf::{ ActionId, ActivityId, AiModel, ApiActivityBuilder, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, Process, SeverityId, StatusId, @@ -36,9 +40,17 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; use tokio::io::{ - AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, AsyncWriteExt, + AsyncBufReadExt, AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, + AsyncWriteExt, }; use tokio::net::{TcpListener, TcpStream}; + +type ProxyClient = tokio::io::BufReader; + +enum ProxyAcceptError { + Listener(std::io::Error), + Source(openshell_isolation::contract::BackendError), +} use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{debug, warn}; @@ -233,6 +245,7 @@ pub struct ProxyHandle { http_addr: Option, join: JoinHandle<()>, exited_rx: Option>, + source_failure: tokio::sync::watch::Receiver>, } impl ProxyHandle { @@ -255,6 +268,7 @@ impl ProxyHandle { activity_tx: Option, engine_ready: tokio::sync::watch::Receiver, upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, + network_mediation_source: Option>, ) -> Result { // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no @@ -270,8 +284,15 @@ impl ProxyHandle { )); } - let listener = TcpListener::bind(http_addr).await.into_diagnostic()?; - let local_addr = listener.local_addr().into_diagnostic()?; + let listener = if network_mediation_source.is_none() { + Some(TcpListener::bind(http_addr).await.into_diagnostic()?) + } else { + None + }; + let local_addr = match listener.as_ref() { + Some(listener) => listener.local_addr().into_diagnostic()?, + None => http_addr, + }; { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Listen) @@ -338,6 +359,7 @@ impl ProxyHandle { } let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>(); + let (source_failure_tx, source_failure) = tokio::sync::watch::channel(None); let join = tokio::spawn(async move { // Hold the sender for the lifetime of this task — when the task // exits (panic, abort, or loop break), the sender drops and the @@ -371,11 +393,34 @@ impl ProxyHandle { let mut consecutive_resource_errors: u32 = 0; let mut consecutive_unknown_errors: u32 = 0; loop { - match listener.accept().await { - Ok((stream, _addr)) => { + let accepted = if let Some(source) = network_mediation_source.as_ref() { + source + .accept() + .await + .map(|connection| { + (connection.stream, Some(connection.binary_identity), None) + }) + .map_err(ProxyAcceptError::Source) + } else { + let listener = listener + .as_ref() + .expect("listener exists without a mediation source"); + listener + .accept() + .await + .map(|(stream, _)| { + set_tcp_nodelay_best_effort(&stream); + let workload_addr = stream.peer_addr().ok(); + let proxy_addr = stream.local_addr().ok(); + let stream: BoundaryDuplexStream = Box::new(stream); + (stream, None, workload_addr.zip(proxy_addr)) + }) + .map_err(ProxyAcceptError::Listener) + }; + match accepted { + Ok((stream, supplied_identity, socket_addrs)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; - set_tcp_nodelay_best_effort(&stream); let opa = opa_engine.clone(); let cache = identity_cache.clone(); let spid = entrypoint_pid.clone(); @@ -398,8 +443,10 @@ impl ProxyHandle { let atx = activity_tx.clone(); tokio::spawn(async move { #[allow(clippy::large_futures)] - if let Err(err) = handle_tcp_connection( - stream, + if let Err(err) = handle_mediated_connection( + tokio::io::BufReader::new(stream), + supplied_identity, + socket_addrs, opa, cache, spid, @@ -427,7 +474,18 @@ impl ProxyHandle { } }); } - Err(err) => { + Err(ProxyAcceptError::Source(err)) => { + let _ = source_failure_tx.send(Some(err.to_string())); + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message(format!("Network-mediation source failed: {err}")) + .build(); + ocsf_emit!(event); + break; + } + Err(ProxyAcceptError::Listener(err)) => { match classify_accept_error( &err, &mut consecutive_resource_errors, @@ -468,6 +526,7 @@ impl ProxyHandle { http_addr: Some(local_addr), join, exited_rx: Some(exited_rx), + source_failure, }) } @@ -479,6 +538,20 @@ impl ProxyHandle { pub fn take_exit_receiver(&mut self) -> Option> { self.exited_rx.take() } + + /// Wait until a backend-provided mediation source fails terminally. + pub async fn wait_for_source_failure(&self) -> String { + let mut receiver = self.source_failure.clone(); + loop { + let current = receiver.borrow().clone(); + if let Some(error) = current { + return error; + } + if receiver.changed().await.is_err() { + return "network mediation stopped".to_string(); + } + } + } } impl Drop for ProxyHandle { @@ -1192,21 +1265,24 @@ fn middleware_uninspectable_gate( Ok(crate::l7::middleware::uninspectable_traffic_gate(&chain)) } -async fn peek_tunnel_protocol(client: &TcpStream) -> Result> { - let mut peek_buf = [0u8; TUNNEL_PROTOCOL_PEEK_BYTES]; +async fn peek_tunnel_protocol(client: &mut C) -> Result> +where + C: tokio::io::AsyncBufRead + Unpin, +{ let deadline = tokio::time::Instant::now() + TUNNEL_PROTOCOL_PEEK_TIMEOUT; loop { - let n = client.peek(&mut peek_buf).await.into_diagnostic()?; - if n == 0 { + let available = client.fill_buf().await.into_diagnostic()?; + if available.is_empty() { return Ok(None); } - let peek = &peek_buf[..n]; + let n = available.len().min(TUNNEL_PROTOCOL_PEEK_BYTES); + let peek = &available[..n]; let protocol = classify_tunnel_protocol(peek); if protocol != TunnelProtocol::Unsupported || !could_be_supported_tunnel_protocol_prefix(peek) - || n == peek_buf.len() + || n == TUNNEL_PROTOCOL_PEEK_BYTES || tokio::time::Instant::now() >= deadline { return Ok(Some(protocol)); @@ -1585,7 +1661,7 @@ fn build_forward_destination_deny_ocsf_event( #[allow(clippy::too_many_arguments)] async fn deny_connect_destination( - client: &mut TcpStream, + client: &mut (impl TokioAsyncWrite + Unpin), denial: &DestinationDenial, peer_addr: SocketAddr, host: &str, @@ -1631,7 +1707,7 @@ async fn deny_connect_destination( #[allow(clippy::too_many_arguments)] async fn deny_forward_destination( - client: &mut TcpStream, + client: &mut (impl TokioAsyncWrite + Unpin), denial: &DestinationDenial, peer_addr: SocketAddr, method: &str, @@ -1681,9 +1757,58 @@ async fn deny_forward_destination( // Many distinct, non-related context parameters are required for a CONNECT // dispatch; bundling them into a struct would just shift the noise into call // sites. +#[cfg(test)] #[allow(clippy::too_many_arguments)] async fn handle_tcp_connection( - mut client: TcpStream, + client: TcpStream, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + tls_state: Option>, + inference_ctx: Option>, + policy_local_ctx: Option>, + agent_proposals: openshell_core::proposals::AgentProposals, + trusted_host_gateway: Arc>, + upstream_proxy: Arc>, + secret_resolver: Option>, + dynamic_credentials: Option< + Arc< + std::sync::RwLock< + std::collections::HashMap, + >, + >, + >, + denial_tx: Option>, + activity_tx: Option, +) -> Result<()> { + let socket_addrs = client.peer_addr().ok().zip(client.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(client); + Box::pin(handle_mediated_connection( + tokio::io::BufReader::new(stream), + None, + socket_addrs, + opa_engine, + identity_cache, + entrypoint_pid, + tls_state, + inference_ctx, + policy_local_ctx, + agent_proposals, + trusted_host_gateway, + upstream_proxy, + secret_resolver, + dynamic_credentials, + denial_tx, + activity_tx, + )) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn handle_mediated_connection( + mut client: ProxyClient, + supplied_identity: Option>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -1757,6 +1882,8 @@ async fn handle_tcp_connection( &buf[..], used, &mut client, + supplied_identity.as_ref(), + socket_addrs, opa_engine, identity_cache, entrypoint_pid, @@ -1803,22 +1930,27 @@ async fn handle_tcp_connection( return Ok(()); } - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); - - // Evaluate OPA policy with process-identity binding. - // Wrapped in spawn_blocking because identity resolution does heavy sync I/O: - // /proc scanning + SHA256 hashing of binaries (e.g. node at 124MB). - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); let intent = EgressIntent::connect(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity.as_ref() { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -2205,7 +2337,7 @@ async fn handle_tcp_connection( // Auto-detect the tunnel payload. L7-configured endpoints must only // enter relays that can enforce their configured protocol; unsupported // bytes fail closed below instead of falling through to raw relay. - let Some(tunnel_protocol) = peek_tunnel_protocol(&client).await? else { + let Some(tunnel_protocol) = peek_tunnel_protocol(&mut client).await? else { return Ok(()); }; @@ -2780,6 +2912,77 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres } } +/// Evaluate an egress intent using identity already bound to the accepted +/// connection by an isolation backend. This is the RFC 0012 path; legacy +/// listeners continue to resolve through procfs in `authorize_egress_intent`. +fn authorize_supplied_identity( + engine: &OpaEngine, + intent: EgressIntent, + identity: &Result, +) -> EgressDecision { + let deny = |reason: String, + binary: Option, + ancestors: Vec, + cmdline_paths: Vec| EgressDecision { + intent: intent.clone(), + action: NetworkAction::Deny { reason }, + policy_generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), + endpoint: EndpointDecision::default(), + binary, + binary_pid: None, + ancestors, + cmdline_paths, + }; + + let identity = match identity { + Ok(identity) => identity, + Err(error) => { + return deny( + format!("backend identity resolution failed: {error}"), + None, + vec![], + vec![], + ); + } + }; + let Some(digest) = identity.binary_digest else { + return deny( + "backend identity did not include the required binary digest".to_string(), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ); + }; + let input = crate::opa::NetworkInput { + host: intent.destination.host.clone(), + port: intent.destination.port, + binary_path: identity.binary_path.clone(), + binary_sha256: digest.to_string(), + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }; + match engine.evaluate_network_action_with_generation(&input) { + Ok((action, generation)) => EgressDecision { + intent, + action, + policy_generation: generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), + binary: Some(identity.binary_path.clone()), + binary_pid: None, + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }, + Err(error) => deny( + format!("policy evaluation error: {error}"), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ), + } +} + /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] fn authorize_egress_intent( @@ -2825,7 +3028,7 @@ const INITIAL_INFERENCE_BUF: usize = 65536; /// Returns [`InferenceOutcome::Routed`] if at least one request was successfully /// routed, or [`InferenceOutcome::Denied`] with a reason for all denial cases. async fn handle_inference_interception( - client: TcpStream, + client: ProxyClient, host: &str, port: u16, tls_state: Option<&Arc>, @@ -3420,7 +3623,7 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette } async fn reject_stale_connect_policy( - client: &mut TcpStream, + client: &mut ProxyClient, host: &str, port: u16, activity_tx: Option<&ActivitySender>, @@ -4818,7 +5021,9 @@ async fn handle_forward_proxy( target_uri: &str, buf: &[u8], used: usize, - client: &mut TcpStream, + client: &mut ProxyClient, + supplied_identity: Option<&Result>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -4922,19 +5127,27 @@ async fn handle_forward_proxy( canonicalize_forward_host_header(&buf[..used], &canonical_authority)?; // 2. Evaluate OPA policy (same identity binding as CONNECT) - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); - - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); let intent = EgressIntent::forward_http(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -6160,7 +6373,7 @@ fn normalize_host(raw_host: &str) -> &str { raw_host.strip_suffix('.').unwrap_or(raw_host) } -async fn respond(client: &mut TcpStream, bytes: &[u8]) -> Result<()> { +async fn respond(client: &mut (impl TokioAsyncWrite + Unpin), bytes: &[u8]) -> Result<()> { client.write_all(bytes).await.into_diagnostic()?; Ok(()) } @@ -6301,7 +6514,7 @@ const TLS_TERMINATION_UNAVAILABLE_DETAIL: &str = "TLS termination unavailable (C /// refused (the caller must stop) and `false` when the caller should proceed to /// establish the tunnel. async fn refuse_connect_when_tls_unavailable( - client: &mut TcpStream, + client: &mut (impl TokioAsyncWrite + Unpin), tls_state_present: bool, effective_tls_skip: bool, ) -> Result { @@ -6497,6 +6710,22 @@ mod tests { } } + struct FailedMediationSource; + + #[async_trait::async_trait] + impl NetworkMediationSource for FailedMediationSource { + async fn accept( + &self, + ) -> std::result::Result< + openshell_isolation::contract::MediatedConnection, + openshell_isolation::contract::BackendError, + > { + Err(openshell_isolation::contract::BackendError::Unavailable( + "test source unavailable".to_string(), + )) + } + } + async fn drive_raw_request_through_handler(raw: Vec) -> Vec { let policy = include_str!("../data/sandbox-policy.rego"); let data = r#" @@ -6541,6 +6770,50 @@ network_policies: {} client.await.unwrap() } + #[tokio::test] + async fn terminal_mediation_source_failure_stops_accepting_fail_static() { + let policy = include_str!("../data/sandbox-policy.rego"); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + policy, + "network_policies: {}", + true, + ) + .expect("engine"), + ); + let (_ready_tx, ready_rx) = tokio::sync::watch::channel(true); + let handle = ProxyHandle::start_with_bind_addr( + &ProxyPolicy { http_addr: None }, + Some(([127, 0, 0, 1], 3128).into()), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(1)), + None, + None, + None, + None, + None, + None, + ready_rx, + &upstream_proxy::UpstreamProxyArgs::default(), + Some(Arc::new(FailedMediationSource)), + ) + .await + .expect("proxy starts before source accept"); + + let failure = tokio::time::timeout( + std::time::Duration::from_secs(1), + handle.wait_for_source_failure(), + ) + .await + .expect("source failure must be observed"); + assert!(failure.contains("test source unavailable")); + assert!( + handle.join.is_finished(), + "accept loop must stop after source loss" + ); + } + #[tokio::test] async fn malformed_forward_headers_are_rejected_before_route_or_middleware_dispatch() { for host in ["api.example.com", "unmatched.example.com"] { @@ -7294,6 +7567,59 @@ network_policies: ); } + #[test] + fn backend_supplied_unresolved_identity_denies_an_allowed_endpoint() { + let policy = include_str!("../data/sandbox-policy.rego"); + let data = r#" +network_policies: + test_allow: + name: test_allow + endpoints: + - { host: api.example.com, port: 443 } + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings_with_binary_identity_required(policy, data, true) + .expect("identity-aware engine"); + let decision = authorize_supplied_identity( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + &Err(ResolveError::NotFound), + ); + + assert!(matches!(decision.action, NetworkAction::Deny { .. })); + assert!(matches!( + decision.identity, + ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed) + )); + } + + #[test] + fn backend_supplied_unresolved_identity_denies_in_endpoint_only_mode() { + let policy = include_str!("../data/sandbox-policy.rego"); + let data = r#" +network_policies: + test_allow: + name: test_allow + endpoints: + - { host: api.example.com, port: 443 } +"#; + let engine = OpaEngine::from_strings_with_binary_identity_required(policy, data, false) + .expect("endpoint-only engine"); + temp_env::with_var( + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + Some("false"), + || { + let decision = authorize_supplied_identity( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + &Err(ResolveError::NotFound), + ); + assert!(matches!(decision.action, NetworkAction::Deny { .. })); + }, + ); + } + fn websocket_l7_config( protocol: crate::l7::L7Protocol, websocket_credential_rewrite: bool, @@ -7354,13 +7680,14 @@ network_policies: let addr = listener.local_addr().unwrap(); let mut client = TcpStream::connect(addr).await.unwrap(); let (server, _) = listener.accept().await.unwrap(); + let mut server = tokio::io::BufReader::new(server); client .write_all(crate::l7::rest::HTTP2_PRIOR_KNOWLEDGE_PREFACE) .await .unwrap(); - let protocol = peek_tunnel_protocol(&server) + let protocol = peek_tunnel_protocol(&mut server) .await .expect("peek should succeed") .expect("client sent bytes"); @@ -10243,9 +10570,10 @@ network_policies: #[tokio::test] async fn test_resolve_check_allowed_ips_rejects_outside_allowlist() { - // 8.8.8.8 resolves to a public IP which is NOT in 10.0.0.0/8 + // A public IP outside 10.0.0.0/8 must be rejected. Use the literal so + // this security check does not depend on external DNS availability. let nets = parse_allowed_ips(&["10.0.0.0/8".to_string()]).unwrap(); - let result = resolve_and_check_allowed_ips("dns.google", 443, &nets, 0).await; + let result = resolve_and_check_allowed_ips("8.8.8.8", 443, &nets, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -11644,9 +11972,9 @@ network_policies: #[tokio::test] async fn test_forward_public_ip_allowed_without_allowed_ips() { - // Public IPs (e.g. dns.google -> 8.8.8.8) should pass through - // resolve_and_reject_internal without needing allowed_ips. - let result = resolve_and_reject_internal("dns.google", 80, 0).await; + // Public IPs should pass through resolve_and_reject_internal without + // needing allowed_ips. Use a literal to keep the test hermetic. + let result = resolve_and_reject_internal("8.8.8.8", 80, 0).await; assert!( result.is_ok(), "Public IP should be allowed without allowed_ips: {result:?}" @@ -11657,7 +11985,7 @@ network_policies: for addr in &addrs { assert!( !is_internal_ip(addr.ip()), - "dns.google should resolve to public IPs, got {}", + "expected a public IP, got {}", addr.ip() ); } diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 2a71702b4b..b50b7800a6 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -37,6 +37,7 @@ use crate::l7::tls::{ use crate::opa::OpaEngine; use crate::policy_local::PolicyLocalContext; use crate::proxy::ProxyHandle; +use openshell_isolation::contract::NetworkMediationSource; #[cfg(target_os = "linux")] pub struct TransparentRuntimeSetup { @@ -196,6 +197,7 @@ pub async fn run_networking( agent_proposals: AgentProposals, workspace_rx: tokio::sync::watch::Receiver, upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, + network_mediation_source: Option>, #[cfg(target_os = "linux")] transparent_runtime: Option, ) -> Result { // Build the policy-local route context. The orchestrator's policy poll @@ -447,6 +449,7 @@ pub async fn run_networking( activity_tx.clone(), engine_ready_rx, upstream_proxy_args, + network_mediation_source, ) .await?; Some(proxy_handle) diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 2b4ea554ed..4f42f10d80 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -53,6 +53,13 @@ pub struct NetworkNamespace { ns_fd: Option, } +/// Cloneable coordinates for continuously verifying a live RFC boundary. +#[derive(Clone, Debug)] +pub struct EgressCeilingVerifier { + namespace: String, + host_ip: IpAddr, +} + impl NetworkNamespace { /// Create a new isolated network namespace with veth pair. /// @@ -249,6 +256,21 @@ impl NetworkNamespace { self.ns_fd } + /// Duplicate the namespace descriptor for a boundary-owned asynchronous + /// operation whose lifetime may outlive this borrow. + pub fn try_clone_ns_fd(&self) -> Result> { + use std::os::fd::FromRawFd; + + let Some(fd) = self.ns_fd else { + return Ok(None); + }; + let duplicated = nix::unistd::dup(fd).into_diagnostic()?; + // nix 0.29 returns a raw descriptor from dup. + Ok(Some(unsafe { + std::os::fd::OwnedFd::from_raw_fd(duplicated) + })) + } + /// Install nftables rules for bypass detection inside the namespace. /// /// Sets up OUTPUT chain rules that: @@ -323,6 +345,30 @@ impl NetworkNamespace { Ok(()) } + /// Install the mandatory RFC default-deny fence. Unlike legacy bypass + /// diagnostics, absence or failure of nftables is fatal. + pub fn install_egress_ceiling(&self, proxy_port: u16) -> Result<()> { + let nft = find_nft().ok_or_else(|| { + miette::miette!("nft not found; cannot establish default-deny egress ceiling") + })?; + let log_prefix = format!("openshell:bypass:{}:", self.name); + enable_nf_log_all_netns(); + let commands = nft_ruleset::generate_egress_ceiling_commands( + &self.host_ip.to_string(), + proxy_port, + Some(&log_prefix), + ); + run_nft_commands_netns(&self.name, &nft, &commands) + } + + #[must_use] + pub fn egress_ceiling_verifier(&self) -> EgressCeilingVerifier { + EgressCeilingVerifier { + namespace: self.name.clone(), + host_ip: self.host_ip, + } + } + /// Replace the ordinary bypass fence with the policy-DNS and transparent /// TCP ruleset. This is fail-closed: callers must not release workload /// execution unless every required rule was installed. @@ -548,6 +594,81 @@ impl NetworkNamespace { } } +impl EgressCeilingVerifier { + /// Read the installed kernel rules under a deadline. Validation requires a + /// policy-drop output chain and explicit proxy/loopback accepts; any other + /// accept in that chain fails closed. + pub async fn verify_bounded( + &self, + proxy_port: u16, + deadline: std::time::Duration, + ) -> Result<()> { + let nft = find_nft().ok_or_else(|| miette::miette!("nft not found"))?; + let nsenter = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; + let net_flag = format!( + "--net={}", + openshell_core::container_paths::netns_path(&self.namespace).display() + ); + let mut command = tokio::process::Command::new(nsenter); + command.kill_on_drop(true).args([ + &net_flag, + "--", + &nft, + "-j", + "list", + "chain", + "inet", + "openshell_bypass", + "output", + ]); + let output = tokio::time::timeout(deadline, command.output()) + .await + .map_err(|_| miette::miette!("egress ceiling verification timed out"))? + .into_diagnostic()?; + if !output.status.success() { + return Err(miette::miette!( + "could not read back egress ceiling: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + verify_egress_ceiling_json(&output.stdout, &self.host_ip.to_string(), proxy_port) + } +} + +fn verify_egress_ceiling_json(json: &[u8], host_ip: &str, proxy_port: u16) -> Result<()> { + let document: serde_json::Value = serde_json::from_slice(json).into_diagnostic()?; + let objects = document + .get("nftables") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| miette::miette!("nft response has no object list"))?; + let default_deny = objects.iter().any(|object| { + object.get("chain").is_some_and(|chain| { + chain.get("family").and_then(serde_json::Value::as_str) == Some("inet") + && chain.get("table").and_then(serde_json::Value::as_str) + == Some("openshell_bypass") + && chain.get("name").and_then(serde_json::Value::as_str) == Some("output") + && chain.get("hook").and_then(serde_json::Value::as_str) == Some("output") + && chain.get("policy").and_then(serde_json::Value::as_str) == Some("drop") + }) + }); + if !default_deny { + return Err(miette::miette!( + "egress ceiling output chain is not policy drop" + )); + } + let rendered = serde_json::to_string(&document).into_diagnostic()?; + if !rendered.contains(host_ip) + || !rendered.contains(&proxy_port.to_string()) + || !rendered.contains("oifname") + || !rendered.contains("lo") + { + return Err(miette::miette!( + "egress ceiling is missing the proxy or loopback allow" + )); + } + Ok(()) +} + impl Drop for NetworkNamespace { fn drop(&mut self) { debug!(namespace = %self.name, "Cleaning up network namespace"); @@ -638,6 +759,27 @@ pub fn create_netns_for_proxy( } } +/// Create the RFC in-pod namespace with mandatory standing egress +/// enforcement. The legacy helper remains best-effort for compatibility. +pub fn create_conformant_netns_for_proxy( + policy: &openshell_core::policy::SandboxPolicy, +) -> Result> { + use openshell_core::policy::NetworkMode; + + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return Ok(None); + } + let namespace = NetworkNamespace::create()?; + let proxy_port = policy + .network + .proxy + .as_ref() + .and_then(|proxy| proxy.http_addr) + .map_or(3128, |address| address.port()); + namespace.install_egress_ceiling(proxy_port)?; + Ok(Some(namespace)) +} + /// Install pod-network bypass enforcement for Kubernetes sidecar topology. /// /// This runs in the current network namespace, not in a per-workload netns. diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 52557493cd..9b815ebbe9 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -2370,6 +2370,16 @@ pub struct ProcessStatus { } impl ProcessStatus { + /// Construct a synthetic normal exit status for supervisor-generated + /// terminal outcomes such as a policy timeout. + #[must_use] + pub const fn exited(code: i32) -> Self { + Self { + code: Some(code), + signal: None, + } + } + /// Get the conventional exit code when the process exited normally. #[must_use] pub const fn exit_code(&self) -> Option { diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index f0e792306d..517167f762 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -57,7 +57,7 @@ fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { /// Returns an error if SSH server startup fails, if the entrypoint child /// fails to spawn, or if waiting for the child returns an OS error. #[allow(clippy::too_many_arguments, clippy::implicit_hasher)] -pub async fn run_process( +pub async fn spawn_workload( program: &str, args: &[String], workspace: ResolvedWorkspace, @@ -83,7 +83,8 @@ pub async fn run_process( tokio::sync::mpsc::UnboundedSender, >, #[cfg(target_os = "linux")] bypass_activity_tx: Option, -) -> Result { + boundary_runtime: Option>, +) -> Result { // Platform drivers with a resolved numeric UID/GID retain the legacy // account-file update. OCI-image identity leaves those environment values // empty, so the image's account files remain unchanged. @@ -266,6 +267,27 @@ pub async fn run_process( let main_session = crate::main_session::MainSession::new(handle.take_io(), main_pid); let main_instance_id = uuid::Uuid::new_v4().to_string(); + #[cfg(target_os = "linux")] + let boundary_netns_fd = netns + .map(NetworkNamespace::try_clone_ns_fd) + .transpose()? + .flatten() + .map(Arc::new); + #[cfg(not(target_os = "linux"))] + let boundary_netns_fd = None; + let boundary_runtime = + boundary_runtime.unwrap_or_else(crate::boundary_io::BoundaryRuntimeState::new); + let port_forward: Arc = + Arc::new(crate::boundary_io::NetnsPortForward::new( + boundary_netns_fd.clone(), + Some(boundary_runtime.clone()), + )); + let user_environment: std::collections::HashMap = + std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) + .ok() + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_default(); + // SSH-spawned shells get http_proxy=http://: exported into // their env so cooperative tools (curl, npm, Node) route through the // CONNECT proxy. Linux uses the netns host_ip; on other targets fall back @@ -275,6 +297,20 @@ pub async fn run_process( #[cfg(not(target_os = "linux"))] let ssh_proxy_url = ssh_proxy_url_for_policy(policy, None); + let boundary_exec: Arc = + Arc::new(crate::boundary_exec::LocalBoundaryExec::new( + policy.clone(), + workspace.owned_root(), + boundary_netns_fd, + ssh_proxy_url.clone(), + ca_file_paths.clone().map(Arc::new), + provider_credentials.clone(), + user_environment, + resolved_process_identity, + enforcement_mode, + boundary_runtime.clone(), + )); + let ssh_socket_path: Option = ssh_socket_path.map(std::path::PathBuf::from); if let Some(listen_path) = ssh_socket_path.clone() { let policy_clone = policy.clone(); @@ -382,6 +418,11 @@ pub async fn run_process( // Store the entrypoint PID so the proxy can resolve TCP peer identity entrypoint_pid.store(handle.pid(), Ordering::Release); + let terminal = Arc::new(AtomicBool::new(false)); + let signal_lock = Arc::new(std::sync::Mutex::new(())); + boundary_runtime + .register_process_group(handle.pid(), terminal.clone(), signal_lock.clone()) + .map_err(|error| miette::miette!(error.to_string()))?; if early_exit.is_none() && let Some(tx) = entrypoint_started_tx { @@ -400,70 +441,250 @@ pub async fn run_process( .build() ); - let outcome = if let Some(status) = early_exit { - ProcessWaitOutcome::Exited(status) - } else { - wait_for_process_exit_or_shutdown(&mut handle, timeout_secs, &supervisor_terminating) - .await? - }; + Ok(SpawnedAgent { + handle, + early_exit, + timeout_secs, + supervisor_terminating, + supervisor_session_task, + main_session, + main_instance_id, + sidecar_exit_tx, + openshell_endpoint: openshell_endpoint.map(str::to_string), + sandbox_id: sandbox_id.map(str::to_string), + boundary_exec, + port_forward, + boundary_runtime, + terminal, + signal_lock, + }) +} - let rendered_code = match outcome { - ProcessWaitOutcome::Exited(status) => status.code(), - ProcessWaitOutcome::TimedOut => { - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Close) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Critical) - .status(StatusId::Failure) - .message("Process timed out, killing") - .build() - ); - 124 - } - ProcessWaitOutcome::ShutdownSignal { signal, status } => { - info!( - signal, - exit_code = status.code(), - "Entrypoint exited after supervisor shutdown signal" - ); - status.code() +/// Run a command through the legacy orchestration entry point. +/// +/// This is intentionally a thin adapter over the owned RFC lifecycle. Existing +/// callers still receive an exit code, while runtime-selectable backends retain +/// the running process and its boundary capabilities. +#[allow(clippy::too_many_arguments, clippy::implicit_hasher)] +pub async fn run_process( + program: &str, + args: &[String], + workspace: ResolvedWorkspace, + timeout_secs: u64, + interactive: bool, + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + ssh_socket_path: Option, + shared_ssh_socket: bool, + ssh_exit_tx: Option>, + policy: &SandboxPolicy, + resolved_process_identity: ResolvedProcessIdentity, + enforcement_mode: ProcessEnforcementMode, + entrypoint_pid: Arc, + entrypoint_started_tx: Option>, + sidecar_exit_tx: Option>, + provider_credentials: ProviderCredentialState, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + agent_proposals: AgentProposals, + #[cfg(target_os = "linux")] netns: Option<&NetworkNamespace>, + #[cfg(target_os = "linux")] bypass_denial_tx: Option< + tokio::sync::mpsc::UnboundedSender, + >, + #[cfg(target_os = "linux")] bypass_activity_tx: Option, +) -> Result { + let mut spawned = spawn_workload( + program, + args, + workspace, + timeout_secs, + interactive, + sandbox_id, + openshell_endpoint, + ssh_socket_path, + shared_ssh_socket, + ssh_exit_tx, + policy, + resolved_process_identity, + enforcement_mode, + entrypoint_pid, + entrypoint_started_tx, + sidecar_exit_tx, + provider_credentials, + provider_env, + ca_file_paths, + agent_proposals, + #[cfg(target_os = "linux")] + netns, + #[cfg(target_os = "linux")] + bypass_denial_tx, + #[cfg(target_os = "linux")] + bypass_activity_tx, + None, + ) + .await?; + Ok(spawned.wait().await?.code()) +} + +/// Owned canonical workload plus the boundary capabilities tied to it. +pub struct SpawnedAgent { + handle: ProcessHandle, + early_exit: Option, + timeout_secs: u64, + supervisor_terminating: Arc, + supervisor_session_task: Option>, + main_session: Arc, + main_instance_id: String, + sidecar_exit_tx: Option>, + openshell_endpoint: Option, + sandbox_id: Option, + boundary_exec: Arc, + port_forward: Arc, + boundary_runtime: Arc, + terminal: Arc, + signal_lock: Arc>, +} + +impl SpawnedAgent { + #[must_use] + pub fn signaler(&self) -> AgentSignaler { + AgentSignaler { + pid: self.handle.pid(), + terminal: self.terminal.clone(), + signal_lock: self.signal_lock.clone(), } - }; - supervisor_terminating.store(true, Ordering::Release); - main_session.finish(rendered_code).await; + } - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Close) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .exit_code(rendered_code) - .message(format!("Process exited with code {rendered_code}")) - .build() - ); + #[must_use] + pub fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } - if let Some(task) = supervisor_session_task { - task.abort(); + #[must_use] + pub fn port_forward(&self) -> Arc { + self.port_forward.clone() } - if let Some(tx) = sidecar_exit_tx { - let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); - tx.send((main_instance_id.clone(), rendered_code, ack_tx)) - .await - .map_err(|_| miette::miette!("sidecar exit reporter closed"))?; - ack_rx - .await - .map_err(|_| miette::miette!("sidecar exit reporter dropped acknowledgement"))? - .map_err(|error| miette::miette!(error))?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code).await; - info!(instance_id = %main_instance_id, "main-process exit acknowledged"); + + #[must_use] + pub fn boundary_runtime(&self) -> Arc { + self.boundary_runtime.clone() } - Ok(rendered_code) + /// Wait for the canonical process and finalize every existing reporting + /// and retained-I/O obligation before publishing the boundary exit. + pub async fn wait(&mut self) -> Result { + let pid = self.handle.pid(); + let outcome = if let Some(status) = self.early_exit.take() { + ProcessWaitOutcome::Exited(status) + } else { + wait_for_process_exit_or_shutdown( + &mut self.handle, + self.timeout_secs, + &self.supervisor_terminating, + ) + .await? + }; + let status = match outcome { + ProcessWaitOutcome::Exited(status) => status, + ProcessWaitOutcome::TimedOut => { + ocsf_emit!( + ProcessActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Close) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Critical) + .status(StatusId::Failure) + .message("Process timed out, killing") + .build() + ); + ProcessStatus::exited(124) + } + ProcessWaitOutcome::ShutdownSignal { signal, status } => { + info!( + signal, + exit_code = status.code(), + "Entrypoint exited after supervisor shutdown signal" + ); + status + } + }; + let rendered_code = status.code(); + self.terminal.store(true, Ordering::Release); + self.boundary_runtime + .unregister_process_group(pid, &self.terminal); + self.boundary_runtime.deactivate(); + self.supervisor_terminating.store(true, Ordering::Release); + self.main_session.finish(rendered_code).await; + + ocsf_emit!( + ProcessActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Close) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .exit_code(rendered_code) + .message(format!("Process exited with code {rendered_code}")) + .build() + ); + if let Some(task) = self.supervisor_session_task.take() { + task.abort(); + } + if let Some(tx) = self.sidecar_exit_tx.take() { + let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); + tx.send((self.main_instance_id.clone(), rendered_code, ack_tx)) + .await + .map_err(|_| miette::miette!("sidecar exit reporter closed"))?; + ack_rx + .await + .map_err(|_| miette::miette!("sidecar exit reporter dropped acknowledgement"))? + .map_err(|error| miette::miette!(error))?; + } else if let (Some(endpoint), Some(id)) = ( + self.openshell_endpoint.as_deref(), + self.sandbox_id.as_deref(), + ) { + report_main_process_exit_until_ack(endpoint, id, &self.main_instance_id, rendered_code) + .await; + info!(instance_id = %self.main_instance_id, "main-process exit acknowledged"); + } + Ok(status) + } +} + +/// Concurrent signal handle for a running canonical agent process group. +#[derive(Clone)] +pub struct AgentSignaler { + pid: u32, + terminal: Arc, + signal_lock: Arc>, +} + +#[cfg(unix)] +impl AgentSignaler { + fn deliver(&self, signal: nix::sys::signal::Signal) -> Result<()> { + let _guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("agent has exited")); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + nix::sys::signal::killpg(nix::unistd::Pid::from_raw(pid), signal).into_diagnostic() + } + + pub fn term(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGTERM) + } + pub fn kill(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGKILL) + } + pub fn interrupt(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGINT) + } + pub fn hangup(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGHUP) + } } async fn report_main_process_exit_until_ack( diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index c77c5c0aff..80c431f6d9 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -6,9 +6,8 @@ # Supervisor image build. # # The final image carries the static `openshell-sandbox` binary used by Docker -# extraction, Podman image volumes, and the Kubernetes init container copy-self -# path. It also includes nftables so the Kubernetes supervisor sidecar can -# install pod-namespace egress enforcement rules. +# extraction, Podman image volumes, and Kubernetes side-loading. It also carries +# a materialized helper runtime used for trusted network setup and enforcement. # # The Rust binary is built natively before this image build runs and staged at: # deploy/docker/.build/prebuilt-binaries//openshell-sandbox @@ -26,7 +25,23 @@ FROM alpine:3.22 AS supervisor ARG TARGETARCH -RUN apk add --no-cache nftables iptables iptables-legacy +RUN apk add --no-cache iproute2 nftables iptables iptables-legacy \ + && mkdir -p /openshell-runtime/usr /openshell-runtime/etc \ + && cp -aL /bin /sbin /lib /openshell-runtime/ \ + && cp -aL /usr/bin /usr/sbin /usr/lib /openshell-runtime/usr/ \ + && if [ -d /etc/iproute2 ]; then \ + cp -aL /etc/iproute2 /openshell-runtime/etc/; \ + fi \ + && if [ -d /usr/share/nftables ]; then \ + mkdir -p /openshell-runtime/usr/share; \ + cp -aL /usr/share/nftables /openshell-runtime/usr/share/; \ + fi \ + && loader="$(find /openshell-runtime/lib -maxdepth 1 -type f -name 'ld-musl-*.so.1' | head -n 1)" \ + && test -n "$loader" \ + && "$loader" --library-path /openshell-runtime/lib:/openshell-runtime/usr/lib \ + /openshell-runtime/sbin/ip -Version \ + && "$loader" --library-path /openshell-runtime/lib:/openshell-runtime/usr/lib \ + /openshell-runtime/usr/sbin/nft --version # --chmod=0555 restores execute bits after the actions/upload-artifact + # download-artifact roundtrip strips them. Ownership stays root (0:0) for diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index d1fed9ae44..3f1fa3da9f 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -563,7 +563,7 @@ the SPIRE OIDC discovery endpoint or its TLS CA. ### Docker -Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. +Sandboxes run as containers on a local bridge network. The driver supplies RFC 0012's co-located backend descriptor by default. The supervisor binary and its trusted network-helper runtime are bind-mounted read-only from driver-controlled sources; guest mTLS material is supplied as host paths. ```toml [openshell] @@ -582,10 +582,11 @@ image_pull_policy = "IfNotPresent" sandbox_namespace = "docker-dev" # Empty auto-detects https://host.openshell.internal: when guest TLS is set. grpc_endpoint = "https://host.openshell.internal:17670" -# Skip the image-pull-and-extract step by pointing at a locally built binary. +# Use a locally built supervisor binary. If it has no sibling +# openshell-runtime directory, the driver still extracts that runtime from supervisor_image. supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" -# When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. -# Defaults to the gateway version; override to pin a specific build. +# Source for /openshell-sandbox and its trusted helper runtime. Defaults to the +# gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" @@ -603,7 +604,7 @@ sandbox_pids_limit = 2048 ### Podman -Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. +Sandboxes run as Podman containers on a user-mode bridge network. The driver supplies RFC 0012's co-located backend descriptor by default. The supervisor image, including its trusted network-helper runtime, is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. ```toml [openshell] @@ -737,7 +738,7 @@ health_check_interval_secs = 10 ### MicroVM -Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. Use this driver when you want stronger isolation than container namespaces alone. +Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. The driver supplies RFC 0012's co-located backend descriptor by default and embeds its trusted network-helper runtime in the guest bootstrap. Use this driver when you want stronger isolation than container namespaces alone. ```toml [openshell] diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 22ba1b039f..a43d36268f 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -281,7 +281,8 @@ if [ ! -d "${COMPRESSED_DIR}" ] \ mise run vm:setup fi -if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then +if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ] \ + || [ ! -f "${COMPRESSED_DIR}/openshell-runtime.tar.zst" ]; then check_supervisor_cross_toolchain echo "==> Building bundled VM supervisor (mise run vm:supervisor)" mise run vm:supervisor diff --git a/tasks/scripts/vm/build-supervisor-bundle.sh b/tasks/scripts/vm/build-supervisor-bundle.sh index 0085c0619d..a31e999603 100755 --- a/tasks/scripts/vm/build-supervisor-bundle.sh +++ b/tasks/scripts/vm/build-supervisor-bundle.sh @@ -60,6 +60,7 @@ esac SUPERVISOR_BIN="${ROOT}/target/${RUST_TARGET}/release/openshell-sandbox" SUPERVISOR_OUTPUT="${OUTPUT_DIR}/openshell-sandbox.zst" +SUPERVISOR_RUNTIME_OUTPUT="${OUTPUT_DIR}/openshell-runtime.tar.zst" echo "==> Building openshell-sandbox supervisor bundle" echo " Guest arch: ${GUEST_ARCH}" @@ -123,6 +124,51 @@ fi zstd -19 -T0 -f "${SUPERVISOR_BIN}" -o "${SUPERVISOR_OUTPUT}" +case "${GUEST_ARCH}" in + aarch64|arm64) DOCKER_ARCH="arm64" ;; + x86_64|amd64) DOCKER_ARCH="amd64" ;; +esac + +echo "==> Building trusted supervisor helper runtime" +STAGED_SUPERVISOR="${ROOT}/deploy/docker/.build/prebuilt-binaries/${DOCKER_ARCH}/openshell-sandbox" +RUNTIME_IMAGE="openshell-vm-helper-runtime:${DOCKER_ARCH}-$$" +mkdir -p "$(dirname "${STAGED_SUPERVISOR}")" +cp "${SUPERVISOR_BIN}" "${STAGED_SUPERVISOR}" + +case "$(uname -m)" in + aarch64|arm64) HOST_DOCKER_ARCH="arm64" ;; + x86_64|amd64) HOST_DOCKER_ARCH="amd64" ;; + *) HOST_DOCKER_ARCH="" ;; +esac + +if [ "${HOST_DOCKER_ARCH}" = "${DOCKER_ARCH}" ]; then + docker build \ + --build-arg "TARGETARCH=${DOCKER_ARCH}" \ + --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ + --tag "${RUNTIME_IMAGE}" \ + "${ROOT}" +else + docker buildx build \ + --load \ + --platform "linux/${DOCKER_ARCH}" \ + --build-arg "TARGETARCH=${DOCKER_ARCH}" \ + --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ + --tag "${RUNTIME_IMAGE}" \ + "${ROOT}" +fi + +RUNTIME_CONTAINER="$(docker create "${RUNTIME_IMAGE}")" +cleanup_runtime_image() { + docker rm -f "${RUNTIME_CONTAINER}" >/dev/null 2>&1 || true + docker image rm "${RUNTIME_IMAGE}" >/dev/null 2>&1 || true +} +trap cleanup_runtime_image EXIT +docker cp "${RUNTIME_CONTAINER}:/openshell-runtime" - \ + | zstd -19 -T0 -f -o "${SUPERVISOR_RUNTIME_OUTPUT}" +cleanup_runtime_image +trap - EXIT + echo "==> Bundled supervisor ready" echo " Binary: $(du -sh "${SUPERVISOR_BIN}" | cut -f1)" echo " Compressed: $(du -sh "${SUPERVISOR_OUTPUT}" | cut -f1)" +echo " Helper runtime: $(du -sh "${SUPERVISOR_RUNTIME_OUTPUT}" | cut -f1)"