diff --git a/Cargo.lock b/Cargo.lock index 0a90c0fc30..4c38a39778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3916,15 +3916,20 @@ dependencies = [ "clap", "futures", "http 1.4.0", + "libc", "miette", "openshell-core", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", "opentelemetry", "opentelemetry_sdk", "prost-types", + "rand 0.9.4", + "rustix 1.1.4", "serde", "serde_json", + "sha2 0.10.9", "tar", "temp-env", "tempfile", diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 7a5fa3fe3d..38051327aa 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-otel = { path = "../openshell-otel" } opentelemetry = { workspace = true } @@ -38,15 +39,19 @@ miette = { workspace = true } toml = { workspace = true } tower-http = { workspace = true } http = { workspace = true } +rand = { workspace = true } +sha2 = { workspace = true } +rustix = { workspace = true } +libc = "0.2" +tar = "0.4" +tempfile = "3" [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } prost-types = { workspace = true } -tar = "0.4" temp-env = "0.3" -tempfile = "3" tracing-subscriber = { workspace = true } [lints] diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index bbd7e69b88..32eb195b4d 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -29,18 +29,35 @@ no candidate responds. ## Runtime Model -The gateway runs as a host process. The Docker driver creates one container per -sandbox and starts the `openshell-sandbox` supervisor inside that container. The -supervisor then creates the nested sandbox namespace for the agent process. +The gateway and Docker driver run on the host. For each sandbox, the driver +starts `openshell-sandbox --mode=control` beside the gateway and +`openshell-sandbox --mode=boundary` as PID 1 in the workload container. The two +roles use the shared RFC 0012 protocol over a private bind-mounted Unix socket. + +Control owns the gateway session, policy, SSH relay, and network proxy. The +boundary owns the process tree, exec/signal/wait/PTY operations, loopback port +forwarding, procfs binary identity, and the workload network namespace. The +boundary receives no gateway JWT, client TLS private key, or policy authority. + +The driver bind-mounts the boundary socket from a short, digest-keyed directory +under `$XDG_RUNTIME_DIR/openshell/docker-boundary-runtime`, falling back to the +XDG state directory when no runtime directory exists. The driver rejects +symlinked, incorrectly owned, or overly permissive pre-existing components and +checks the final path against the Unix-socket length limit. The full protected +topology and gateway material remain in the normal driver state directory, and +sandbox cleanup removes both locations. ## Stop and Start -Stop stops the managed container without removing it. Docker retains the -container writable layer, attached volumes, labels, token material, and restart -policy. Start starts that same container, so files in the resolved OCI -workspace remain available. A durably stopped sandbox is excluded from -gateway startup recovery and stays stopped across gateway restarts. Delete -continues to force-remove the container and clean up driver-owned material. +Stop terminates host control and stops the managed container without removing +it. Docker retains the container writable layer, attached volumes, labels, and +the protected driver topology. Stop removes the transient boundary and SSH +socket names; Start validates and removes any safe stale sockets before starting +that same container and recreating host control. Files in the resolved OCI +workspace remain available. A +durably stopped sandbox is excluded from gateway startup recovery and stays +stopped across gateway restarts. Delete force-removes the container and the +driver-owned boundary directory. Graceful gateway shutdown sends `StopSandbox` for each sandbox whose persisted phase requires running compute without changing that persisted intent. On startup, the gateway sends an idempotent `StartSandbox` request for the same @@ -50,9 +67,9 @@ remain excluded. Before creating the container, the driver inspects the final sandbox image and captures its immutable image ID, raw OCI `Config.User`, and OCI `Config.WorkingDir`. Container creation uses that image ID, preventing a -mutable tag from changing between inspection and launch. The supervisor runs as -root, resolves omitted policy identity fields from the image declaration, and -drops only agent children to the resulting identity. Named OCI components +mutable tag from changing between inspection and launch. The boundary runs as +root, resolves omitted policy identity fields from the image declaration inside +the image filesystem, and drops only agent children to the resulting identity. Named OCI components remain names after validation; a missing group is filled with the user's numeric primary GID. Explicit `process.run_as_user` and `process.run_as_group` values take precedence independently. @@ -83,14 +100,12 @@ are rejected, as are paths that overlap concrete OpenShell control resources. The workspace is the child cwd and `HOME`. The supervisor starts from `/`, then reports an invalid workdir as a readiness failure. -Docker containers join an OpenShell-managed bridge network. The driver injects -`host.openshell.internal` and `host.docker.internal` so supervisors have stable -names for reaching the gateway host. On Docker Desktop, Colima, Rancher -Desktop, OrbStack, and macOS-hosted gateways, those names use Docker's -`host-gateway` alias. The driver requests a separate IPv4 loopback callback -listener when the primary listener does not already cover it. On native Linux -Docker, the gateway also binds the bridge gateway IP so containers can call -back to the host process. +Boundary containers use Docker `network_mode=none`. The nested agent network +namespace can reach only the boundary's mediation listener. Each accepted TCP +stream and its procfs-derived executable identity cross the private Unix socket +to host control, which applies policy before dialing the destination. Host +control reaches the gateway directly; the existing managed bridge listener is +retained for compatibility with legacy containers created before this split. ## Container Contract @@ -100,13 +115,15 @@ contract: | Setting | Purpose | |---|---| | `user = "0"` | The supervisor needs root inside the container to prepare namespaces, mounts, Landlock, and seccomp. | -| `network_mode = openshell` | Places the supervisor on the managed Docker bridge network. | +| `network_mode = none` | Removes direct container egress; the nested workload namespace can emit only through the authenticated boundary stream. | | `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | | `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | | `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | | `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Set `[openshell.drivers.docker].sandbox_pids_limit = 0` to inherit the Docker/runtime default. | | CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | -| `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | +| Read-only supervisor/runtime mounts | Supplies the boundary binary and trusted `ip`/`nft` helper runtime without trusting workload-image tools. | +| Private boundary bind mount | Carries the protected boundary config and Unix control socket. The directory is host-owned mode `0700`; workload children cannot access it. | +| `policy-dns-transparent-tcp` capability | Declares that the shared boundary can own namespace-local DNS/TCP capture while host control retains authorization, pinned dialing, relaying, and OCSF decisions. The marker is not exposed to the workload environment. | The agent child process does not retain these supervisor privileges. @@ -127,8 +144,9 @@ mount types: Host bind mounts are disabled by default because they expose gateway host paths to sandbox requests. Image mounts are not part of the Docker -driver-config schema. The driver still uses internal bind mounts for -OpenShell-owned supervisor, token, and TLS material. +driver-config schema. The driver still uses internal bind mounts for the +OpenShell-owned boundary binary, trusted helper runtime, and private socket +directory. Gateway token and TLS material remain on the host. Docker `bind` mounts accept `source`, `target`, optional `read_only`, and an optional `selinux_label` of `shared` (applies `:z`) or `private` (applies @@ -154,8 +172,9 @@ openshell sandbox create \ ## Supervisor Binary Resolution -The Docker driver bind-mounts a host-side Linux `openshell-sandbox` binary into -each sandbox container. Resolution order is: +The Docker driver bind-mounts a host-side Linux `openshell-sandbox` boundary +binary and trusted helper runtime into each sandbox container. Resolution order +for the Linux artifact is: 1. `supervisor_bin` in `[openshell.drivers.docker]`. 2. `supervisor_image` in `[openshell.drivers.docker]`, extracting @@ -164,22 +183,32 @@ each sandbox container. Resolution order is: 4. A local Linux cargo target build for the Docker daemon architecture. 5. The release-matched default supervisor image, extracting `/openshell-sandbox`. +When a selected binary has no valid sibling `openshell-runtime`, the driver +extracts `/openshell-runtime` from the configured or release-matched supervisor +image. Host control uses the native `openshell-sandbox` beside the gateway (or a +local native cargo build); Linux hosts can reuse the boundary binary. + Release and Docker-image gateway builds bake the matching supervisor image tag into the binary at compile time. The default Docker supervisor image is not `:latest` unless a custom build explicitly sets that tag. ## Callback and TLS -`OPENSHELL_ENDPOINT` is injected from the gateway's configured gRPC endpoint. +`OPENSHELL_ENDPOINT` is injected only into host control from the gateway's +configured gRPC endpoint. When no endpoint is configured, the driver uses `host.openshell.internal:` with the appropriate HTTP or HTTPS scheme. Set `host_gateway_ip` only when the host has an explicit, locally assigned address that containers should use for callbacks; package-managed -macOS gateways should leave it unset. +macOS gateways should leave it unset. The current boundary topology requires +the Docker daemon and gateway to share bind-mounted Unix socket inodes. The +driver fails during initialization for Docker Desktop, Colima, Lima, Rancher +Desktop, OrbStack, and other detected VM-backed daemons instead of starting a +workload whose host control channel cannot become operational. For HTTPS endpoints, the server certificate must include the endpoint host as a -subject alternative name. Docker sandboxes also need the client TLS bundle -mounted into the container and exposed with: +subject alternative name. The client TLS bundle is exposed only to host control +with: - `OPENSHELL_TLS_CA` - `OPENSHELL_TLS_CERT` @@ -189,8 +218,12 @@ HTTP endpoints reject TLS material because the supervisor would not use it. ## Environment Ownership -The driver merges template environment and sandbox spec environment first, then -overwrites security-critical keys: +The driver writes template and sandbox environment into the protected boundary +config. Boundary mode exposes that map only to workload children. Host control +starts with an empty inherited environment and receives only driver-owned +values. It also arms Linux `PDEATHSIG` with a parent-PID race check so a killed +gateway cannot leave an orphan host-control process. Host control owns these +security-critical keys: - `OPENSHELL_ENDPOINT` - `OPENSHELL_SANDBOX_ID` diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs new file mode 100644 index 0000000000..a1011e8793 --- /dev/null +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Docker provisioning for the shared authenticated boundary protocol. +//! +//! Docker owns only the container/socket topology and immutable OCI resource +//! claims. Lifecycle, process, network, identity, and wire behavior live in +//! `openshell-isolation-interface` and `openshell-sandbox`. + +use std::collections::{BTreeMap, HashMap}; +use std::net::IpAddr; +use std::path::PathBuf; + +use openshell_isolation_interface::boundary_protocol::{ + BOUNDARY_PROTOCOL_VERSION, BoundaryAgentIdentity, BoundaryConfig, BoundaryListener, + BoundaryTopology, BoundaryTransport, +}; + +/// Driver-owned inputs that bind one Docker container to one boundary. +pub struct DockerBoundarySpec { + pub boundary_id: String, + pub bootstrap_token: String, + pub container_id: String, + pub image_identity: String, + pub listener_socket: PathBuf, + pub control_socket: PathBuf, + pub host_gateway_ip: Option, + pub oci_user: String, + pub trusted_runtime_root: PathBuf, + pub child_env: HashMap, +} + +/// Protected container config and matching host descriptor. +pub struct DockerBoundaryProvisioning { + pub boundary_config: BoundaryConfig, + pub topology: BoundaryTopology, +} + +impl DockerBoundarySpec { + /// Produce both sides of the common protocol from the same immutable + /// Docker coordinates so attach cannot bind a different container. + #[must_use] + pub fn provision(self) -> DockerBoundaryProvisioning { + let resource_claims = BTreeMap::from([ + ("docker.container_id".to_string(), self.container_id), + ("docker.image_identity".to_string(), self.image_identity), + ]); + DockerBoundaryProvisioning { + boundary_config: BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: self.boundary_id.clone(), + bootstrap_token: self.bootstrap_token.clone(), + listener: BoundaryListener::Unix { + socket_path: self.listener_socket, + }, + resource_claims: resource_claims.clone(), + agent_identity: BoundaryAgentIdentity::OciUser { + declaration: self.oci_user, + }, + protect_config_file: true, + trusted_runtime_root: self.trusted_runtime_root, + child_env: self.child_env, + }, + topology: BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: self.boundary_id, + transport: BoundaryTransport::Unix { + socket_path: self.control_socket, + }, + host_gateway_ip: self.host_gateway_ip, + resource_claims, + bootstrap_token: self.bootstrap_token, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provisioning_binds_container_and_image_claims() { + let provisioned = DockerBoundarySpec { + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "a".repeat(64), + container_id: "sha256:container".to_string(), + image_identity: "sha256:image".to_string(), + listener_socket: PathBuf::from("/run/openshell/boundary/control.sock"), + control_socket: PathBuf::from("/host/control.sock"), + host_gateway_ip: Some(IpAddr::from([127, 0, 0, 1])), + oci_user: "1000:1000".to_string(), + trusted_runtime_root: PathBuf::from("/opt/openshell/bin/openshell-runtime"), + child_env: HashMap::new(), + } + .provision(); + + assert_eq!( + provisioned.boundary_config.resource_claims, + provisioned.topology.resource_claims + ); + assert_eq!( + provisioned.topology.resource_claims["docker.container_id"], + "sha256:container" + ); + } +} diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d599cac169..ae22715cdf 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5,15 +5,16 @@ #![allow(clippy::result_large_err)] +mod isolation; pub mod otel_tracing; use bollard::Docker; use bollard::errors::Error as BollardError; use bollard::models::{ ContainerCreateBody, ContainerState, ContainerStateStatusEnum, ContainerSummary, - ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, EndpointSettings, HostConfig, Mount, - MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, NetworkCreateRequest, NetworkingConfig, - ProgressDetail, SystemInfo, + ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, HostConfig, Mount, + MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, NetworkCreateRequest, ProgressDetail, + SystemInfo, }; use bollard::query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, @@ -55,16 +56,23 @@ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; use openshell_core::{Error, Result as CoreResult}; +use openshell_isolation_interface::boundary_protocol::BoundaryTopology; use opentelemetry::trace::TraceContextExt as _; +use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +#[cfg(unix)] +use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _, PermissionsExt as _}; use std::path::{Path, PathBuf}; use std::pin::Pin; +use std::process::Stdio; use std::sync::Arc; use std::task::{Context, Poll}; use std::time::Duration; -use tokio::sync::{Mutex, broadcast, mpsc}; +use tokio::process::Command; +use tokio::sync::{Mutex, broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; @@ -77,11 +85,18 @@ const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; -const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; -const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; -const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; -const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; -const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; +const SUPERVISOR_RUNTIME_MOUNT_PATH: &str = "/opt/openshell/bin/openshell-runtime"; +const SUPERVISOR_IMAGE_RUNTIME_PATH: &str = "/openshell-runtime"; +const BOUNDARY_MOUNT_PATH: &str = "/run/openshell/boundary"; +const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/run/openshell/boundary/config.json"; +const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/run/openshell/boundary/control.sock"; +const DOCKER_BOUNDARY_RUNTIME_DIR: &str = "docker-boundary-runtime"; +const DRIVER_ADMITTED_BACKEND: &str = "docker"; +const LABEL_ISOLATION_TOPOLOGY: &str = "openshell.ai/isolation-topology"; +const LABEL_ISOLATION_TOPOLOGY_BOUNDARY_V1: &str = "boundary-v1"; +const TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload"; +const MAIN_PROCESS_SPEC_FILE: &str = "main-process.json"; +const WORKSPACE_ROOT_FILE: &str = "workspace-root"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; const DOCKER_NETWORK_DRIVER: &str = "bridge"; @@ -200,14 +215,17 @@ struct DockerDriverRuntimeConfig { default_image: String, image_pull_policy: String, sandbox_namespace: String, - grpc_endpoint: String, - network_name: String, gateway_route: DockerGatewayRoute, gateway_callback_bind_address: Option, ssh_socket_path: String, stop_timeout_secs: u32, log_level: String, supervisor_bin: PathBuf, + supervisor_runtime: PathBuf, + control_bin: PathBuf, + boundary_runtime_root: PathBuf, + host_grpc_endpoint: String, + gateway_tls_server_name: Option, guest_tls: Option, daemon_version: String, supports_gpu: bool, @@ -218,10 +236,7 @@ struct DockerDriverRuntimeConfig { #[derive(Debug, Clone, PartialEq, Eq)] enum DockerGatewayRoute { - Bridge { - bind_address: SocketAddr, - host_alias_ip: IpAddr, - }, + Bridge { bind_address: SocketAddr }, HostGateway, } @@ -233,6 +248,24 @@ pub struct DockerComputeDriver { pending: Arc>>, gpu_selector: Arc, lifecycle_event_fences: DockerLifecycleEventFences, + control_processes: Arc>>, + control_failures: Arc>>, +} + +struct DockerControlProcess { + shutdown: Option>, + task: JoinHandle<()>, +} + +#[derive(Clone)] +struct DockerControlFailureContext { + docker: Arc, + events: broadcast::Sender, + failures: Arc>>, + sandbox: DriverSandbox, + sandbox_namespace: String, + container_id: String, + stop_timeout_secs: u32, } /// Per-sandbox container exit timestamps that fence snapshots from an earlier run. @@ -583,6 +616,7 @@ impl DockerComputeDriver { let info = docker.info().await.map_err(|err| { Error::execution(format!("failed to query Docker daemon info: {err}")) })?; + validate_shared_unix_boundary_transport(&info, cfg!(target_os = "macos"))?; let supports_gpu = info .cdi_spec_dirs .as_ref() @@ -613,14 +647,34 @@ impl DockerComputeDriver { docker_config.grpc_endpoint = format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); } - let grpc_endpoint = docker_container_openshell_endpoint( - &docker_config.grpc_endpoint, - HOST_OPENSHELL_INTERNAL, - gateway_port, - ); + let host_grpc_endpoint = + docker_host_openshell_endpoint(&docker_config.grpc_endpoint, &gateway_route)?; + let original_gateway_url = Url::parse(&docker_config.grpc_endpoint).map_err(|error| { + Error::config(format!( + "invalid docker grpc_endpoint '{}': {error}", + docker_config.grpc_endpoint + )) + })?; + let host_gateway_url = Url::parse(&host_grpc_endpoint).map_err(|error| { + Error::config(format!( + "invalid normalized Docker host grpc_endpoint '{host_grpc_endpoint}': {error}" + )) + })?; + let gateway_tls_server_name = (original_gateway_url.scheme() == "https" + && original_gateway_url.host_str() != host_gateway_url.host_str()) + .then(|| { + original_gateway_url + .host_str() + .unwrap_or_default() + .to_string() + }); let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; + let supervisor_runtime = + resolve_supervisor_runtime(&docker, &docker_config, &supervisor_bin).await?; + let control_bin = resolve_control_supervisor_bin(&supervisor_bin)?; let guest_tls = docker_guest_tls_paths(&docker_config)?; + let boundary_runtime_root = prepare_docker_boundary_runtime_root()?; let driver = Self { docker: Arc::new(docker), @@ -628,14 +682,17 @@ impl DockerComputeDriver { default_image: docker_config.default_image.clone(), image_pull_policy: docker_config.image_pull_policy.clone(), sandbox_namespace: docker_config.sandbox_namespace.clone(), - grpc_endpoint, - network_name, gateway_route, gateway_callback_bind_address, ssh_socket_path: docker_config.ssh_socket_path.clone(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: gateway_log_level.to_string(), supervisor_bin, + supervisor_runtime, + control_bin, + boundary_runtime_root, + host_grpc_endpoint, + gateway_tls_server_name, guest_tls, daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), supports_gpu, @@ -650,6 +707,8 @@ impl DockerComputeDriver { allow_all_default_gpu, )), lifecycle_event_fences: DockerLifecycleEventFences::default(), + control_processes: Arc::new(Mutex::new(HashMap::new())), + control_failures: Arc::new(Mutex::new(HashMap::new())), }; let poll_driver = driver.clone(); @@ -851,9 +910,10 @@ impl DockerComputeDriver { let container = self .find_managed_container_summary(sandbox_id, sandbox_name) .await?; - if let Some(sandbox) = + if let Some(mut sandbox) = container.and_then(|summary| sandbox_from_container_summary(&summary)) { + self.apply_control_failure(&mut sandbox).await; return Ok(Some(sandbox)); } @@ -891,6 +951,7 @@ impl DockerComputeDriver { } } } + self.apply_control_failure(&mut sandbox).await; container_sandboxes.push(sandbox); } let mut by_id = self.pending_snapshot_map().await; @@ -1007,11 +1068,21 @@ impl DockerComputeDriver { image.ref = %template.image, )) .await?; + prepare_docker_boundary_state_dir(sandbox, &self.config).map_err(|status| { + DockerProvisioningFailure::new("BoundaryStateCreateFailed", status.message()) + })?; let token_file_created = write_sandbox_token_file(sandbox, &self.config) .await .map_err(|status| { DockerProvisioningFailure::new("SandboxTokenWriteFailed", status.message()) })?; + if !token_file_created { + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "SandboxTokenWriteFailed", + "Docker control mode requires a gateway sandbox token", + )); + } let container_name = container_name_for_sandbox(sandbox); let gpu_devices = self @@ -1022,9 +1093,7 @@ impl DockerComputeDriver { ) .await .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } + cleanup_docker_boundary_state(sandbox, &self.config); DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; let create_body = build_container_create_body_for_image( @@ -1035,12 +1104,10 @@ impl DockerComputeDriver { &image, ) .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } + cleanup_docker_boundary_state(sandbox, &self.config); DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; - async { + let created = async { openshell_otel::record_error_result( self.docker .create_container( @@ -1053,9 +1120,7 @@ impl DockerComputeDriver { ) .await .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } + cleanup_docker_boundary_state(sandbox, &self.config); DockerProvisioningFailure::from_status( "ContainerCreateFailed", create_status_from_docker_error("create docker sandbox container", err), @@ -1078,6 +1143,25 @@ impl DockerComputeDriver { HashMap::from([("container_name".to_string(), container_name.clone())]), ); + let topology = + match prepare_docker_boundary_files(sandbox, &self.config, &created.id, &image).await { + Ok(topology) => topology, + Err(status) => { + let _ = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "BoundaryConfigWriteFailed", + status.message(), + )); + } + }; + let start_result = async { openshell_otel::record_error_result( self.docker.start_container(&container_name, None).await, @@ -1107,14 +1191,35 @@ impl DockerComputeDriver { "Failed to clean up Docker container after start failure" ); } - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } + cleanup_docker_boundary_state(sandbox, &self.config); return Err(DockerProvisioningFailure::from_status( "ContainerStartFailed", create_status_from_docker_error("start docker sandbox container", err), )); } + self.clear_control_failure(&sandbox.id).await; + let failure_context = self.control_failure_context(sandbox.clone(), created.id.clone()); + let control = + match spawn_docker_control_process(sandbox, &self.config, &topology, failure_context) + .await + { + Ok(control) => control, + Err(status) => { + let _ = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "ControlSupervisorStartFailed", + status.message(), + )); + } + }; + self.replace_control_process(&sandbox.id, control).await; self.publish_docker_progress( &sandbox.id, "Started", @@ -1135,6 +1240,119 @@ impl DockerComputeDriver { span_status.finish(Ok(())) } + async fn replace_control_process(&self, sandbox_id: &str, process: DockerControlProcess) { + let previous = self + .control_processes + .lock() + .await + .insert(sandbox_id.to_string(), process); + if let Some(previous) = previous { + stop_docker_control_process(previous).await; + } + } + + async fn clear_control_failure(&self, sandbox_id: &str) { + self.control_failures.lock().await.remove(sandbox_id); + } + + fn control_failure_context( + &self, + sandbox: DriverSandbox, + container_id: String, + ) -> DockerControlFailureContext { + DockerControlFailureContext { + docker: self.docker.clone(), + events: self.events.clone(), + failures: self.control_failures.clone(), + sandbox, + sandbox_namespace: self.config.sandbox_namespace.clone(), + container_id, + stop_timeout_secs: self.config.stop_timeout_secs, + } + } + + async fn apply_control_failure(&self, sandbox: &mut DriverSandbox) { + let message = self.control_failures.lock().await.get(&sandbox.id).cloned(); + if let Some(message) = message { + set_sandbox_ready_condition( + sandbox, + error_condition("ControlSupervisorExited", &message), + ); + } + } + + async fn stop_control_process(&self, sandbox_id: &str) { + let process = self.control_processes.lock().await.remove(sandbox_id); + if let Some(process) = process { + stop_docker_control_process(process).await; + } + } + + async fn ensure_control_process_for_container( + &self, + container: &ContainerSummary, + ) -> Result<(), Status> { + let Some(sandbox) = sandbox_from_container_summary(container) else { + return Err(Status::internal( + "managed Docker container is missing sandbox identity labels", + )); + }; + let stale = { + let mut processes = self.control_processes.lock().await; + match processes.get(&sandbox.id) { + Some(process) if !process.task.is_finished() => return Ok(()), + Some(_) => processes.remove(&sandbox.id), + None => None, + } + }; + if let Some(stale) = stale { + stop_docker_control_process(stale).await; + } + let Some(topology) = read_docker_boundary_topology(&sandbox.id, &self.config).await? else { + if container_requires_boundary_control(container) { + let container_id = summary_container_target(container).ok_or_else(|| { + Status::internal("managed Docker container has no id or name") + })?; + let failure_context = self.control_failure_context(sandbox.clone(), container_id); + let status = Status::failed_precondition( + "Docker boundary topology is missing; refusing to leave the workload running without host control", + ); + handle_docker_control_failure(failure_context, status.message().to_string()).await; + return Err(status); + } + // A container created by the legacy combined supervisor has no + // boundary marker or topology file and remains self-managing. + return Ok(()); + }; + let container_id = summary_container_target(container) + .ok_or_else(|| Status::internal("managed Docker container has no id or name"))?; + self.clear_control_failure(&sandbox.id).await; + let failure_context = self.control_failure_context(sandbox.clone(), container_id); + let process = match spawn_docker_control_process( + &sandbox, + &self.config, + &topology, + failure_context.clone(), + ) + .await + { + Ok(process) => process, + Err(status) => { + handle_docker_control_failure( + failure_context, + format!( + "failed to start Docker control supervisor: {}", + status.message() + ), + ) + .await; + return Err(status); + } + }; + self.replace_control_process(&sandbox.id, process).await; + Ok(()) + } + async fn delete_sandbox_inner( &self, sandbox_id: &str, @@ -1146,6 +1364,9 @@ impl DockerComputeDriver { { task.abort(); } + if let Some(record) = pending.as_ref() { + self.stop_control_process(&record.sandbox.id).await; + } let Some(container) = self .find_managed_container_summary(sandbox_id, sandbox_name) @@ -1162,11 +1383,13 @@ impl DockerComputeDriver { .await { Ok(()) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); + self.clear_control_failure(&record.sandbox.id).await; + cleanup_docker_boundary_state(&record.sandbox, &self.config); return Ok(true); } Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); + self.clear_control_failure(&record.sandbox.id).await; + cleanup_docker_boundary_state(&record.sandbox, &self.config); return Ok(true); } Err(err) => { @@ -1179,6 +1402,12 @@ impl DockerComputeDriver { let Some(target) = summary_container_target(&container) else { return Ok(pending.is_some()); }; + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + self.stop_control_process(resolved_sandbox_id).await; match self .docker @@ -1189,11 +1418,13 @@ impl DockerComputeDriver { .await { Ok(()) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + self.clear_control_failure(resolved_sandbox_id).await; + cleanup_docker_boundary_state_by_id(resolved_sandbox_id, &self.config); Ok(true) } Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + self.clear_control_failure(resolved_sandbox_id).await; + cleanup_docker_boundary_state_by_id(resolved_sandbox_id, &self.config); Ok(pending.is_some()) } Err(err) => Err(internal_status("delete docker sandbox container", err)), @@ -1206,10 +1437,12 @@ impl DockerComputeDriver { .await? else { if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { + self.stop_control_process(&record.sandbox.id).await; + self.clear_control_failure(&record.sandbox.id).await; if let Some(task) = record.task { task.abort(); } - cleanup_sandbox_token_file(&record.sandbox, &self.config); + cleanup_docker_boundary_state(&record.sandbox, &self.config); self.publish_deleted(record.sandbox.id); return Ok(()); } @@ -1218,8 +1451,14 @@ impl DockerComputeDriver { let Some(target) = summary_container_target(&container) else { return Err(Status::not_found("sandbox container has no id or name")); }; + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + self.stop_control_process(resolved_sandbox_id).await; - match self + let result = match self .docker .stop_container( &target, @@ -1235,7 +1474,12 @@ impl DockerComputeDriver { Err(err) if is_not_modified_error(&err) => Ok(()), Err(err) if is_not_found_error(&err) => Err(Status::not_found("sandbox not found")), Err(err) => Err(internal_status("stop docker sandbox container", err)), + }; + if result.is_ok() { + self.clear_control_failure(resolved_sandbox_id).await; + cleanup_docker_boundary_sockets_by_id(resolved_sandbox_id, &self.config); } + result } /// Start a managed sandbox container that was previously stopped. Used @@ -1287,6 +1531,8 @@ impl DockerComputeDriver { }; let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); if !container_state_needs_start(state) { + self.ensure_control_process_for_container(&container) + .await?; return Ok(true); } @@ -1310,14 +1556,34 @@ impl DockerComputeDriver { self.lifecycle_event_fences .record_previous_exit(sandbox_id, previous_finished_at.as_deref()); + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + if container_requires_boundary_control(&container) { + if read_docker_boundary_topology(resolved_sandbox_id, &self.config) + .await? + .is_none() + { + return Err(Status::failed_precondition( + "Docker boundary topology is missing; refusing to start the workload without host control", + )); + } + prepare_docker_boundary_restart_sockets(resolved_sandbox_id, &self.config)?; + } + match self.docker.start_container(&target, None).await { - Ok(()) => Ok(true), + Ok(()) => {} // Already running — race with another start path or the // restart policy. Treat as success. - Err(err) if is_not_modified_error(&err) => Ok(true), - Err(err) if is_not_found_error(&err) => Ok(false), - Err(err) => Err(internal_status("start docker sandbox container", err)), + Err(err) if is_not_modified_error(&err) => {} + Err(err) if is_not_found_error(&err) => return Ok(false), + Err(err) => return Err(internal_status("start docker sandbox container", err)), } + self.ensure_control_process_for_container(&container) + .await?; + Ok(true) } async fn reserve_pending_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), Status> { @@ -1386,7 +1652,7 @@ impl DockerComputeDriver { sandbox: &DriverSandbox, failure: &DockerProvisioningFailure, ) { - cleanup_sandbox_token_file(sandbox, &self.config); + cleanup_docker_boundary_state(sandbox, &self.config); let snapshot = pending_sandbox_snapshot( sandbox, &self.config.sandbox_namespace, @@ -1423,8 +1689,9 @@ impl DockerComputeDriver { if let Some(summary) = self .find_managed_container_summary(sandbox_id, sandbox_name) .await? - && let Some(sandbox) = sandbox_from_container_summary(&summary) + && let Some(mut sandbox) = sandbox_from_container_summary(&summary) { + self.apply_control_failure(&mut sandbox).await; self.publish_sandbox_snapshot(sandbox); } Ok(()) @@ -2267,6 +2534,21 @@ fn error_condition(reason: &str, message: &str) -> DriverCondition { } } +fn set_sandbox_ready_condition(sandbox: &mut DriverSandbox, condition: DriverCondition) { + let Some(status) = sandbox.status.as_mut() else { + return; + }; + if let Some(existing) = status + .conditions + .iter_mut() + .find(|existing| existing.r#type == "Ready") + { + *existing = condition; + } else { + status.conditions.push(condition); + } +} + fn platform_event( source: &str, event_type: &str, @@ -2688,36 +2970,205 @@ fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { }) } -fn build_binds( +fn build_binds(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { + vec![ + format!( + "{}:{}:ro,z", + config.supervisor_bin.display(), + SUPERVISOR_MOUNT_PATH + ), + format!( + "{}:{}:ro,z", + config.supervisor_runtime.display(), + SUPERVISOR_RUNTIME_MOUNT_PATH + ), + format!( + "{}:{}:rw,z", + docker_boundary_mount_dir(sandbox, config).display(), + BOUNDARY_MOUNT_PATH + ), + ] +} + +fn docker_boundary_state_dir( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, -) -> Result, Status> { - let mut binds = vec![format!( - "{}:{}:ro,z", - config.supervisor_bin.display(), - SUPERVISOR_MOUNT_PATH - )]; - if let Some(tls) = &config.guest_tls { - binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); - binds.push(format!( - "{}:{}:ro,z", - tls.cert.display(), - TLS_CERT_MOUNT_PATH +) -> Result { + docker_boundary_state_dir_by_id(&sandbox.id, config) +} + +fn docker_boundary_state_dir_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result { + sandbox_token_host_path_by_id(sandbox_id, config).and_then(|path| { + path.parent() + .map(Path::to_path_buf) + .ok_or_else(|| Status::internal("docker boundary state path has no parent")) + }) +} + +fn docker_boundary_mount_dir( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> PathBuf { + docker_boundary_mount_dir_by_id(&sandbox.id, config) +} + +fn docker_boundary_mount_dir_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> PathBuf { + let mut hasher = Sha256::new(); + hasher.update(config.sandbox_namespace.as_bytes()); + hasher.update([0]); + hasher.update(sandbox_id.as_bytes()); + let digest = format!("{:x}", hasher.finalize()); + config.boundary_runtime_root.join(&digest[..32]) +} + +fn prepare_docker_boundary_runtime_root() -> CoreResult { + let (base_root, require_private_base) = if let Some(runtime_root) = + std::env::var_os("XDG_RUNTIME_DIR").filter(|value| !value.is_empty()) + { + (PathBuf::from(runtime_root), true) + } else { + ( + openshell_core::paths::xdg_state_dir().map_err(|error| { + Error::config(format!("resolve Docker boundary state directory: {error}")) + })?, + false, + ) + }; + std::fs::create_dir_all(&base_root).map_err(|error| { + Error::config(format!( + "create Docker boundary runtime base {}: {error}", + base_root.display() + )) + })?; + validate_owned_directory(&base_root, require_private_base).map_err(Error::config)?; + let openshell_root = base_root.join("openshell"); + ensure_owned_directory(&openshell_root, false).map_err(Error::config)?; + + let runtime_root = openshell_root.join(DOCKER_BOUNDARY_RUNTIME_DIR); + ensure_owned_directory(&runtime_root, true).map_err(Error::config)?; + validate_docker_socket_path_length(&runtime_root).map_err(Error::config)?; + Ok(runtime_root) +} + +fn validate_docker_socket_path_length(runtime_root: &Path) -> Result<(), String> { + let longest = runtime_root.join("f".repeat(32)).with_extension("ssh"); + #[cfg(unix)] + let length = { + use std::os::unix::ffi::OsStrExt as _; + longest.as_os_str().as_bytes().len() + }; + #[cfg(not(unix))] + let length = longest.as_os_str().to_string_lossy().len(); + if length >= 108 { + return Err(format!( + "Docker boundary runtime path '{}' is too long for a Unix socket; set XDG_RUNTIME_DIR or XDG_STATE_HOME to a shorter private path", + runtime_root.display() )); - binds.push(format!("{}:{}:ro,z", tls.key.display(), TLS_KEY_MOUNT_PATH)); } - if sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.is_empty()) - { - binds.push(format!( - "{}:{}:ro,z", - sandbox_token_host_path(sandbox, config)?.display(), - SANDBOX_TOKEN_MOUNT_PATH + Ok(()) +} + +fn ensure_owned_directory(path: &Path, require_private: bool) -> Result<(), String> { + match std::fs::create_dir(path) { + Ok(()) => { + #[cfg(unix)] + std::fs::set_permissions( + path, + std::fs::Permissions::from_mode(if require_private { 0o700 } else { 0o755 }), + ) + .map_err(|error| format!("restrict directory {}: {error}", path.display()))?; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(format!("create directory {}: {error}", path.display())), + } + validate_owned_directory(path, require_private) +} + +fn validate_owned_directory(path: &Path, require_private: bool) -> Result<(), String> { + let metadata = std::fs::symlink_metadata(path) + .map_err(|error| format!("inspect directory {}: {error}", path.display()))?; + if metadata.file_type().is_symlink() || !metadata.file_type().is_dir() { + return Err(format!( + "Docker boundary path '{}' must be a directory and must not be a symlink", + path.display() )); } - Ok(binds) + #[cfg(unix)] + { + let expected_uid = rustix::process::geteuid().as_raw(); + if metadata.uid() != expected_uid { + return Err(format!( + "Docker boundary path '{}' is owned by uid {}, expected {}", + path.display(), + metadata.uid(), + expected_uid + )); + } + let mode = metadata.mode() & 0o777; + let invalid_mode = if require_private { + mode != 0o700 + } else { + mode & 0o022 != 0 + }; + if invalid_mode { + return Err(format!( + "Docker boundary path '{}' has unsafe mode {mode:04o}", + path.display() + )); + } + } + Ok(()) +} + +fn remove_owned_stale_socket( + path: &Path, + expected_mode: u32, + allow_root_owner: bool, +) -> Result<(), String> { + match std::fs::symlink_metadata(path) { + #[cfg(unix)] + Ok(metadata) + if metadata.file_type().is_socket() + && !metadata.file_type().is_symlink() + && (metadata.uid() == rustix::process::geteuid().as_raw() + || (allow_root_owner && metadata.uid() == 0)) + && metadata.mode() & 0o777 == expected_mode => + { + std::fs::remove_file(path) + .map_err(|error| format!("remove stale socket {}: {error}", path.display())) + } + #[cfg(not(unix))] + Ok(metadata) if !metadata.file_type().is_symlink() && metadata.file_type().is_file() => { + std::fs::remove_file(path) + .map_err(|error| format!("remove stale socket {}: {error}", path.display())) + } + Ok(_) => Err(format!( + "Docker socket '{}' exists but is not an expected owned mode-{expected_mode:04o} Unix socket", + path.display(), + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(format!("inspect socket {}: {error}", path.display())), + } +} + +fn docker_control_ssh_socket( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> PathBuf { + docker_control_ssh_socket_by_id(&sandbox.id, config) +} + +fn docker_control_ssh_socket_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> PathBuf { + docker_boundary_mount_dir_by_id(sandbox_id, config).with_extension("ssh") } fn sandbox_token_host_path( @@ -2779,164 +3230,480 @@ async fn write_sandbox_token_file( Ok(true) } -fn cleanup_sandbox_token_file(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { - cleanup_sandbox_token_file_by_id(&sandbox.id, config); +fn prepare_docker_boundary_state_dir( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + openshell_core::paths::create_dir_restricted(&directory).map_err(|error| { + Status::internal(format!( + "create Docker boundary state directory {}: {error}", + directory.display() + )) + })?; + let boundary_directory = docker_boundary_mount_dir(sandbox, config); + validate_owned_directory(&config.boundary_runtime_root, true).map_err(|error| { + Status::failed_precondition(format!("validate Docker boundary runtime root: {error}")) + })?; + ensure_owned_directory(&boundary_directory, true).map_err(|error| { + Status::internal(format!( + "create Docker boundary mount directory {}: {error}", + boundary_directory.display() + )) + })?; + let socket = boundary_directory.join("control.sock"); + remove_owned_stale_socket(&socket, 0o666, true).map_err(Status::failed_precondition)?; + remove_owned_stale_socket(&docker_control_ssh_socket(sandbox, config), 0o600, false) + .map_err(Status::failed_precondition)?; + Ok(directory) } -fn cleanup_sandbox_token_file_for_delete( - sandbox_id: &str, - pending: Option<&PendingSandboxRecord>, +async fn write_docker_boundary_file(path: &Path, contents: &[u8]) -> Result<(), Status> { + tokio::fs::write(path, contents).await.map_err(|error| { + Status::internal(format!( + "write Docker boundary file {}: {error}", + path.display() + )) + })?; + openshell_core::paths::set_file_owner_only(path).map_err(|error| { + Status::internal(format!( + "restrict Docker boundary file {}: {error}", + path.display() + )) + }) +} + +async fn prepare_docker_boundary_files( + sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, -) { - if !sandbox_id.is_empty() { - cleanup_sandbox_token_file_by_id(sandbox_id, config); - } else if let Some(record) = pending { - cleanup_sandbox_token_file(&record.sandbox, config); - } + container_id: &str, + image: &DockerImageMetadata, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + let boundary_directory = docker_boundary_mount_dir(sandbox, config); + let bootstrap_token = random_boundary_token(); + let host_gateway_ip = Some(match config.gateway_route { + DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), + DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), + }); + let provisioning = isolation::DockerBoundarySpec { + boundary_id: sandbox.id.clone(), + bootstrap_token, + container_id: container_id.to_string(), + image_identity: image.id.clone(), + listener_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH), + control_socket: boundary_directory.join("control.sock"), + host_gateway_ip, + oci_user: image.user.clone(), + trusted_runtime_root: PathBuf::from(SUPERVISOR_RUNTIME_MOUNT_PATH), + child_env: docker_child_environment(sandbox), + } + .provision(); + write_docker_boundary_file( + &boundary_directory.join("config.json"), + &provisioning + .boundary_config + .encode() + .map_err(|error| Status::internal(error.to_string()))?, + ) + .await?; + let descriptor = provisioning + .topology + .descriptor(DRIVER_ADMITTED_BACKEND) + .map_err(|error| Status::internal(error.to_string()))?; + write_docker_boundary_file(&directory.join(TOPOLOGY_PAYLOAD_FILE), &descriptor.payload).await?; + let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( + sandbox.spec.as_ref(), + ) + .map_err(|error| Status::internal(format!("encode Docker main process spec: {error}")))?; + write_docker_boundary_file( + &directory.join(MAIN_PROCESS_SPEC_FILE), + main_process_spec.as_bytes(), + ) + .await?; + let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) + .map_err(Status::failed_precondition)?; + write_docker_boundary_file( + &directory.join(WORKSPACE_ROOT_FILE), + workspace_root.as_bytes(), + ) + .await?; + Ok(provisioning.topology) } -fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { - let Ok(path) = sandbox_token_host_path_by_id(sandbox_id, config) else { - return; +async fn read_docker_boundary_topology( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result, Status> { + let path = docker_boundary_state_dir_by_id(sandbox_id, config)?.join(TOPOLOGY_PAYLOAD_FILE); + let bytes = match tokio::fs::read(&path).await { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(Status::internal(format!( + "read Docker boundary topology {}: {error}", + path.display() + ))); + } }; - if let Err(err) = std::fs::remove_file(&path) - && err.kind() != std::io::ErrorKind::NotFound - { - warn!( - sandbox_id = %sandbox_id, - path = %path.display(), - error = %err, - "Failed to remove Docker sandbox token file" + serde_json::from_slice(&bytes).map(Some).map_err(|error| { + Status::internal(format!( + "decode Docker boundary topology {}: {error}", + path.display() + )) + }) +} + +async fn spawn_docker_control_process( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + topology: &BoundaryTopology, + failure_context: DockerControlFailureContext, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + let descriptor = topology + .descriptor(DRIVER_ADMITTED_BACKEND) + .map_err(|error| Status::internal(error.to_string()))?; + let main_process_spec = tokio::fs::read_to_string(directory.join(MAIN_PROCESS_SPEC_FILE)) + .await + .map_err(|error| Status::internal(format!("read Docker main process spec: {error}")))?; + let workspace_root = tokio::fs::read_to_string(directory.join(WORKSPACE_ROOT_FILE)) + .await + .map_err(|error| Status::internal(format!("read Docker workspace root: {error}")))?; + let stdout = std::fs::File::create(directory.join("supervisor.log")) + .map_err(|error| Status::internal(format!("create Docker supervisor log: {error}")))?; + let stderr = std::fs::File::create(directory.join("supervisor.err.log")).map_err(|error| { + Status::internal(format!("create Docker supervisor error log: {error}")) + })?; + let mut command = new_control_command(&config.control_bin); + command + .kill_on_drop(true) + .stdin(Stdio::null()) + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)) + .arg("--mode=control") + .arg(format!( + "--topology-backend-name={}", + descriptor.backend_name + )) + .arg(format!("--topology-version={}", descriptor.version)) + .arg("--topology-payload-file") + .arg(directory.join(TOPOLOGY_PAYLOAD_FILE)) + .arg("--workdir") + .arg(workspace_root) + .env( + openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND, + DRIVER_ADMITTED_BACKEND, + ) + .env( + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, + main_process_spec, + ) + .env( + openshell_core::sandbox_env::ENDPOINT, + &config.host_grpc_endpoint, + ) + .env(openshell_core::sandbox_env::SANDBOX_ID, &sandbox.id) + .env(openshell_core::sandbox_env::SANDBOX, &sandbox.name) + .env( + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, + sandbox_token_host_path(sandbox, config)?, + ) + .env( + openshell_core::sandbox_env::SSH_SOCKET_PATH, + docker_control_ssh_socket(sandbox, config), + ) + .env( + openshell_core::sandbox_env::PROXY_TLS_DIR, + directory.join("proxy-tls"), + ) + .env(openshell_core::sandbox_env::SANDBOX_UID, "") + .env(openshell_core::sandbox_env::SANDBOX_GID, "") + .env(openshell_core::sandbox_env::OCI_IMAGE_USER, "") + .env( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY, + ) + .env( + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), + ) + .env( + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value(), + ); + if let Some(server_name) = config.gateway_tls_server_name.as_deref() { + command.env( + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + server_name, ); } - if let Some(dir) = path.parent() { - let _ = std::fs::remove_dir(dir); + if let Some(tls) = &config.guest_tls { + command + .env(openshell_core::sandbox_env::TLS_CA, &tls.ca) + .env(openshell_core::sandbox_env::TLS_CERT, &tls.cert) + .env(openshell_core::sandbox_env::TLS_KEY, &tls.key); } + let mut child = command.spawn().map_err(|error| { + Status::internal(format!( + "start Docker control supervisor {}: {error}", + config.control_bin.display() + )) + })?; + let sandbox_id = sandbox.id.clone(); + let (shutdown, mut shutdown_requested) = oneshot::channel(); + let task = tokio::spawn(async move { + tokio::select! { + biased; + _ = &mut shutdown_requested => { + if let Err(error) = child.start_kill() { + warn!(%sandbox_id, %error, "Failed to terminate Docker control supervisor"); + } + let _ = child.wait().await; + } + result = child.wait() => match result { + Ok(status) => { + warn!(%sandbox_id, %status, "Docker control supervisor exited unexpectedly"); + handle_docker_control_failure( + failure_context, + format!("Docker control supervisor exited with status {status}"), + ).await; + } + Err(error) => { + warn!(%sandbox_id, %error, "Failed to wait for Docker control supervisor"); + handle_docker_control_failure( + failure_context, + format!("failed to wait for Docker control supervisor: {error}"), + ).await; + } + }, + } + }); + Ok(DockerControlProcess { + shutdown: Some(shutdown), + task, + }) } -#[cfg(test)] -fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { - build_environment_for_oci_user(sandbox, config, "") +fn new_control_command(binary: &Path) -> Command { + let mut command = Command::new(binary); + command.env_clear(); + configure_control_parent_death(&mut command); + command } -fn build_environment_for_oci_user( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - oci_user: &str, -) -> Vec { - let mut environment = HashMap::from([ - ("HOME".to_string(), "/root".to_string()), - ("PATH".to_string(), SUPERVISOR_PATH.to_string()), - ("TERM".to_string(), "xterm".to_string()), - ( - "OPENSHELL_LOG_LEVEL".to_string(), - openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), +#[cfg(target_os = "linux")] +fn configure_control_parent_death(command: &mut Command) { + let expected_parent = std::process::id(); + #[allow(unsafe_code)] + // SAFETY: the closure only invokes async-signal-safe syscalls and creates + // OS errors from integer codes between fork and exec. + unsafe { + command.pre_exec(move || { + if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 { + return Err(std::io::Error::last_os_error()); + } + if libc::getppid().cast_unsigned() != expected_parent { + return Err(std::io::Error::from_raw_os_error(libc::ECHILD)); + } + Ok(()) + }); + } +} + +#[cfg(not(target_os = "linux"))] +fn configure_control_parent_death(_command: &mut Command) {} + +async fn handle_docker_control_failure(context: DockerControlFailureContext, message: String) { + context + .failures + .lock() + .await + .insert(context.sandbox.id.clone(), message.clone()); + + let mut snapshot = pending_sandbox_snapshot( + &context.sandbox, + &context.sandbox_namespace, + error_condition("ControlSupervisorExited", &message), + false, + ); + if let Some(status) = snapshot.status.as_mut() { + status.instance_id.clone_from(&context.container_id); + } + let _ = context.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(snapshot), + }, + )), + }); + let _ = context.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::PlatformEvent( + WatchSandboxesPlatformEvent { + sandbox_id: context.sandbox.id.clone(), + event: Some(platform_event( + "docker", + "Warning", + "ControlSupervisorExited", + format!("{message}; stopping the isolated workload container"), + )), + }, + )), + }); + + match context + .docker + .stop_container( + &context.container_id, + Some( + StopContainerOptionsBuilder::default() + .t(docker_stop_timeout_secs(context.stop_timeout_secs)) + .build(), + ), + ) + .await + { + Ok(()) => info!( + sandbox_id = %context.sandbox.id, + container_id = %context.container_id, + "Stopped Docker sandbox after control supervisor failure" ), - ]); + Err(error) if is_not_found_error(&error) || is_not_modified_error(&error) => {} + Err(error) => warn!( + sandbox_id = %context.sandbox.id, + container_id = %context.container_id, + %error, + "Failed to stop Docker sandbox after control supervisor failure" + ), + } +} - if let Some(spec) = sandbox.spec.as_ref() { - let mut user_env = HashMap::new(); - if let Some(template) = spec.template.as_ref() { - user_env.extend(template.environment.clone()); - } - user_env.extend(spec.environment.clone()); - environment.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - environment.insert( - openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), - json, - ); - } +async fn stop_docker_control_process(mut process: DockerControlProcess) { + if let Some(shutdown) = process.shutdown.take() { + let _ = shutdown.send(()); } + let _ = process.task.await; +} - environment.insert( - openshell_core::sandbox_env::ENDPOINT.to_string(), - config.grpc_endpoint.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_ID.to_string(), - sandbox.id.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX.to_string(), - sandbox.name.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), - config.ssh_socket_path.clone(), - ); - let main_process = - openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(sandbox.spec.as_ref()) - .expect("main process config serialization cannot fail"); - environment.insert( - openshell_core::sandbox_env::MAIN_PROCESS_SPEC.to_string(), - main_process, - ); - environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - openshell_core::telemetry::enabled_env_value().to_string(), - ); - environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), - ); - // The root supervisor executes namespace helpers during bootstrap; keep - // their search path driver-owned even when the template/spec set PATH. - environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); - if config.guest_tls.is_some() { - environment.insert( - openshell_core::sandbox_env::TLS_CA.to_string(), - TLS_CA_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_CERT.to_string(), - TLS_CERT_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_KEY.to_string(), - TLS_KEY_MOUNT_PATH.to_string(), +fn prepare_docker_boundary_restart_sockets( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result<(), Status> { + let boundary_directory = docker_boundary_mount_dir_by_id(sandbox_id, config); + validate_owned_directory(&config.boundary_runtime_root, true).map_err(|error| { + Status::failed_precondition(format!("validate Docker boundary runtime root: {error}")) + })?; + validate_owned_directory(&boundary_directory, true).map_err(|error| { + Status::failed_precondition(format!("validate Docker boundary directory: {error}")) + })?; + remove_owned_stale_socket(&boundary_directory.join("control.sock"), 0o666, true) + .map_err(Status::failed_precondition)?; + remove_owned_stale_socket( + &docker_control_ssh_socket_by_id(sandbox_id, config), + 0o600, + false, + ) + .map_err(Status::failed_precondition) +} + +fn cleanup_docker_boundary_sockets_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { + if let Err(error) = prepare_docker_boundary_restart_sockets(sandbox_id, config) { + warn!( + %sandbox_id, + %error, + "Failed to clean up stopped Docker boundary sockets" ); } +} - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - // Prevent user-supplied environment from overriding the TLS server name - // the supervisor verifies — a sandbox user who can redirect the gateway - // hostname could otherwise present a certificate for a name they control - // and intercept the sandbox JWT. - environment.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - environment.insert( - openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), - oci_user.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_UID.to_string(), - String::new(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_GID.to_string(), - String::new(), - ); +fn cleanup_docker_boundary_state(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { + cleanup_docker_boundary_state_by_id(&sandbox.id, config); +} - // Gateway-minted sandbox JWT. Keep the raw bearer out of container - // metadata; the supervisor reads it from this driver-owned bind mount. - if let Some(spec) = sandbox.spec.as_ref() - && !spec.sandbox_token.is_empty() +fn cleanup_docker_boundary_state_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { + let Ok(directory) = docker_boundary_state_dir_by_id(sandbox_id, config) else { + return; + }; + let boundary_directory = docker_boundary_mount_dir_by_id(sandbox_id, config); + cleanup_docker_boundary_sockets_by_id(sandbox_id, config); + if let Err(error) = std::fs::remove_dir_all(&boundary_directory) + && error.kind() != std::io::ErrorKind::NotFound { - environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), - SANDBOX_TOKEN_MOUNT_PATH.to_string(), + warn!( + %sandbox_id, + path = %boundary_directory.display(), + %error, + "Failed to remove Docker boundary runtime directory" ); } + if let Err(error) = std::fs::remove_dir_all(&directory) + && error.kind() != std::io::ErrorKind::NotFound + { + warn!( + %sandbox_id, + path = %directory.display(), + %error, + "Failed to remove Docker boundary state directory" + ); + } +} - let mut pairs = environment.into_iter().collect::>(); - pairs.sort_by(|left, right| left.0.cmp(&right.0)); - pairs - .into_iter() - .map(|(key, value)| format!("{key}={value}")) - .collect() +fn random_boundary_token() -> String { + let mut token = String::with_capacity(64); + for byte in rand::random::<[u8; 32]>() { + write!(&mut token, "{byte:02x}").expect("writing to String cannot fail"); + } + token +} + +fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap { + let mut environment = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .map_or_else(HashMap::new, |template| template.environment.clone()); + if let Some(spec) = sandbox.spec.as_ref() { + environment.extend(spec.environment.clone()); + } + for protected in [ + openshell_core::sandbox_env::ENDPOINT, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX, + openshell_core::sandbox_env::SANDBOX_GID, + openshell_core::sandbox_env::SANDBOX_ID, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, + openshell_core::sandbox_env::SANDBOX_UID, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + openshell_core::sandbox_env::TLS_CA, + openshell_core::sandbox_env::TLS_CERT, + openshell_core::sandbox_env::TLS_KEY, + openshell_core::sandbox_env::USER_ENVIRONMENT, + ] { + environment.remove(protected); + } + environment +} + +fn build_boundary_environment( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Vec { + vec![ + format!( + "{}={}", + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level) + ), + format!( + "{}={}", + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value() + ), + ] } fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory { @@ -3044,6 +3811,8 @@ fn build_container_create_body_for_image( .map_err(Status::failed_precondition)?; driver_mounts::validate_workspace_control_path(&workspace_root, &config.ssh_socket_path) .map_err(Status::failed_precondition)?; + driver_mounts::validate_workspace_control_path(&workspace_root, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; for volume in &image.volumes { driver_mounts::validate_container_mount_target(volume).map_err(|error| { Status::failed_precondition(format!( @@ -3057,6 +3826,8 @@ fn build_container_create_body_for_image( })?; driver_mounts::validate_mount_control_path(volume, &config.ssh_socket_path) .map_err(Status::failed_precondition)?; + driver_mounts::validate_mount_control_path(volume, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; } for mount in &driver_config.mounts { let target = match mount { @@ -3069,6 +3840,8 @@ fn build_container_create_body_for_image( .map_err(Status::failed_precondition)?; driver_mounts::validate_mount_control_path(target, &config.ssh_socket_path) .map_err(Status::failed_precondition)?; + driver_mounts::validate_mount_control_path(target, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; } let user_mounts = docker_driver_mounts(driver_config)?; let user_bind_strings = docker_driver_bind_strings(driver_config)?; @@ -3098,6 +3871,10 @@ fn build_container_create_body_for_image( LABEL_SANDBOX_NAMESPACE.to_string(), config.sandbox_namespace.clone(), ); + labels.insert( + LABEL_ISOLATION_TOPOLOGY.to_string(), + LABEL_ISOLATION_TOPOLOGY_BOUNDARY_V1.to_string(), + ); Ok(ContainerCreateBody { image: Some(image.id.clone()), @@ -3105,11 +3882,14 @@ fn build_container_create_body_for_image( // The image workspace may need to be created or rejected by the // supervisor, so do not let the OCI runtime chdir there first. working_dir: Some("/".to_string()), - env: Some(build_environment_for_oci_user(sandbox, config, &image.user)), + env: Some(build_boundary_environment(sandbox, config)), entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), - // Replace the image CMD with the supervisor's resolved workspace - // argument so Docker cannot append inherited image arguments. - cmd: Some(vec!["--workdir".to_string(), workspace_root]), + // The image cannot append inherited arguments or select either role. + cmd: Some(vec![ + "--mode=boundary".to_string(), + "--boundary-config".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ]), labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, @@ -3117,7 +3897,7 @@ fn build_container_create_body_for_image( pids_limit: docker_pids_limit(config.sandbox_pids_limit)?, device_requests, binds: { - let mut binds = build_binds(sandbox, config)?; + let mut binds = build_binds(sandbox, config); binds.extend(user_bind_strings); Some(binds) }, @@ -3142,16 +3922,23 @@ fn build_container_create_body_for_image( // container layer is redundant relative to those controls // and conflicts with them in this case. security_opt: Some(vec!["apparmor=unconfined".to_string()]), - network_mode: Some(config.network_name.clone()), - extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), + // The container itself has no Docker network. The boundary role + // gives only the nested workload namespace an egress path, and + // carries each accepted stream to host control over the private + // Unix socket. + network_mode: Some("none".to_string()), + // `network_mode=none` disables Docker's embedded DNS, but the + // stable host aliases are still part of the driver contract. Ask + // Docker to materialize them in `/etc/hosts`; workload traffic + // remains fenced to the boundary proxy, while libc lookups and + // proxy clients retain the portable hostname identity. + extra_hosts: Some(vec![ + format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), + format!("{HOST_DOCKER_INTERNAL}:host-gateway"), + ]), ..Default::default() }), - networking_config: Some(NetworkingConfig { - endpoints_config: Some(HashMap::from([( - config.network_name.clone(), - EndpointSettings::default(), - )])), - }), + networking_config: None, ..Default::default() }) } @@ -3170,16 +3957,41 @@ fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<() Ok(()) } -fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); - }; +fn container_requires_boundary_control(container: &ContainerSummary) -> bool { + container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_TOPOLOGY)) + .is_some_and(|value| value == LABEL_ISOLATION_TOPOLOGY_BOUNDARY_V1) + || container.command.as_deref().is_some_and(|command| { + command + .split_ascii_whitespace() + .any(|argument| argument == "--mode=boundary") + }) +} - if url.set_host(Some(host)).is_ok() && url.set_port(Some(port)).is_ok() { - return url.to_string(); +fn docker_host_openshell_endpoint( + endpoint: &str, + route: &DockerGatewayRoute, +) -> CoreResult { + let mut url = Url::parse(endpoint) + .map_err(|error| Error::config(format!("invalid docker grpc_endpoint: {error}")))?; + if !matches!( + url.host_str(), + Some(HOST_OPENSHELL_INTERNAL | HOST_DOCKER_INTERNAL) + ) { + return Ok(url.to_string()); } - - endpoint.to_string() + let host = match route { + DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), + DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), + }; + url.set_host(Some(&host.to_string())).map_err(|error| { + Error::config(format!( + "failed to map Docker gateway alias to its host listener: {error}" + )) + })?; + Ok(url.to_string()) } fn docker_network_name(config: &DockerComputeConfig) -> String { @@ -3227,7 +4039,6 @@ fn docker_gateway_route_for_host( if let Some(host_alias_ip) = host_gateway_ip { return DockerGatewayRoute::Bridge { bind_address: SocketAddr::new(host_alias_ip, port), - host_alias_ip, }; } @@ -3236,7 +4047,6 @@ fn docker_gateway_route_for_host( } else { DockerGatewayRoute::Bridge { bind_address: SocketAddr::new(bridge_gateway_ip, port), - host_alias_ip: bridge_gateway_ip, } } } @@ -3261,6 +4071,18 @@ fn host_runtime_requires_host_gateway_alias() -> bool { cfg!(target_os = "macos") } +fn validate_shared_unix_boundary_transport( + info: &SystemInfo, + host_requires_host_gateway_alias: bool, +) -> CoreResult<()> { + if host_requires_host_gateway_alias || uses_host_gateway_alias(info) { + return Err(Error::config( + "Docker boundary control requires a local daemon that shares host Unix sockets; Docker Desktop, Colima, Lima, Rancher Desktop, OrbStack, and other VM-backed Docker daemons are not supported by this topology", + )); + } + Ok(()) +} + /// Detect Docker Desktop and behaviourally compatible runtimes - Colima, /// Lima, Rancher Desktop, and `OrbStack` - that share Docker Desktop's routing /// constraint: the bridge gateway IP is reachable from inside containers but @@ -3301,19 +4123,6 @@ fn uses_host_gateway_alias(info: &SystemInfo) -> bool { }) } -fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { - match route { - DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ - format!("{HOST_DOCKER_INTERNAL}:{host_alias_ip}"), - format!("{HOST_OPENSHELL_INTERNAL}:{host_alias_ip}"), - ], - DockerGatewayRoute::HostGateway => vec![ - format!("{HOST_DOCKER_INTERNAL}:host-gateway"), - format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), - ], - } -} - async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { match docker.inspect_network(network_name, None).await { Ok(network) => return validate_bridge_network(network_name, &network), @@ -3873,18 +4682,75 @@ pub(crate) async fn resolve_supervisor_bin( } } +fn resolve_control_supervisor_bin(boundary_bin: &Path) -> CoreResult { + if let Ok(current_exe) = std::env::current_exe() + && let Some(parent) = current_exe.parent() + { + let sibling = parent.join("openshell-sandbox"); + if sibling.is_file() { + return canonicalize_existing_file(&sibling, "Docker host control supervisor"); + } + } + for candidate in [ + docker_workspace_root().join("target/release/openshell-sandbox"), + docker_workspace_root().join("target/debug/openshell-sandbox"), + ] { + if candidate.is_file() { + return canonicalize_existing_file(&candidate, "Docker host control supervisor"); + } + } + if cfg!(target_os = "linux") { + return Ok(boundary_bin.to_path_buf()); + } + Err(Error::config( + "Docker control mode requires a native openshell-sandbox binary beside the gateway", + )) +} + +async fn resolve_supervisor_runtime( + docker: &Docker, + docker_config: &DockerComputeConfig, + supervisor_bin: &Path, +) -> CoreResult { + if let Some(runtime) = supervisor_bin + .parent() + .map(|parent| parent.join("openshell-runtime")) + && validate_supervisor_runtime(&runtime).is_ok() + { + return Ok(runtime); + } + + let image = docker_config + .supervisor_image + .clone() + .unwrap_or_else(openshell_core::config::default_supervisor_image); + let extracted_bin = extract_supervisor_bin_from_image(docker, &image).await?; + let runtime = extracted_bin + .parent() + .expect("cache path has a parent") + .join("openshell-runtime"); + validate_supervisor_runtime(&runtime)?; + Ok(runtime) +} + fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { match daemon_arch { - "arm64" => vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )], - "amd64" => vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )], + "arm64" => vec![ + docker_workspace_root() + .join("target/aarch64-unknown-linux-gnu/release/openshell-sandbox"), + ], + "amd64" => vec![ + docker_workspace_root() + .join("target/x86_64-unknown-linux-gnu/release/openshell-sandbox"), + ], _ => Vec::new(), } } +fn docker_workspace_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + /// Pull the supervisor image (if not already local), extract /// `/openshell-sandbox` to a host cache keyed by the image's content /// digest, and return the cache path. @@ -3945,6 +4811,8 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core .map_err(Error::config)?; if cache_path.is_file() { validate_linux_elf_binary(&cache_path).map_err(Error::config)?; + ensure_cached_supervisor_runtime(docker, image, cache_path.parent().expect("cache parent")) + .await?; return Ok(cache_path); } @@ -3958,9 +4826,95 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; write_cache_binary_atomic(&cache_path, &binary_bytes).map_err(Error::config)?; validate_linux_elf_binary(&cache_path).map_err(Error::config)?; + ensure_cached_supervisor_runtime(docker, image, cache_path.parent().expect("cache parent")) + .await?; Ok(cache_path) } +fn validate_supervisor_runtime(runtime: &Path) -> CoreResult<()> { + if !runtime.is_dir() { + return Err(Error::config(format!( + "trusted supervisor helper runtime '{}' is missing", + runtime.display() + ))); + } + let has_ip = ["usr/sbin/ip", "sbin/ip", "usr/bin/ip", "bin/ip"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_nft = ["usr/sbin/nft", "sbin/nft", "usr/bin/nft"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_loader = std::fs::read_dir(runtime.join("lib")) + .ok() + .into_iter() + .flatten() + .filter_map(Result::ok) + .any(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("ld-musl-") && name.ends_with(".so.1")) + }); + if !has_ip || !has_nft || !has_loader { + return Err(Error::config(format!( + "trusted supervisor helper runtime '{}' is incomplete", + runtime.display() + ))); + } + Ok(()) +} + +async fn ensure_cached_supervisor_runtime( + docker: &Docker, + image: &str, + cache_dir: &Path, +) -> CoreResult<()> { + let runtime = cache_dir.join("openshell-runtime"); + if validate_supervisor_runtime(&runtime).is_ok() { + return Ok(()); + } + + let archive = extract_supervisor_runtime_archive(docker, image).await?; + let staging = tempfile::Builder::new() + .prefix(".openshell-runtime-") + .tempdir_in(cache_dir) + .map_err(|error| Error::config(format!("create runtime staging directory: {error}")))?; + let mut archive = tar::Archive::new(std::io::Cursor::new(archive)); + for entry in archive + .entries() + .map_err(|error| Error::config(format!("open supervisor runtime archive: {error}")))? + { + let mut entry = entry + .map_err(|error| Error::config(format!("read supervisor runtime archive: {error}")))?; + let kind = entry.header().entry_type(); + if !kind.is_file() && !kind.is_dir() { + return Err(Error::config( + "supervisor runtime archive contains a link or special file", + )); + } + if !entry.unpack_in(staging.path()).map_err(|error| { + Error::config(format!("extract supervisor runtime archive: {error}")) + })? { + return Err(Error::config( + "supervisor runtime archive contains a path outside its root", + )); + } + } + let extracted = staging.path().join("openshell-runtime"); + validate_supervisor_runtime(&extracted)?; + match std::fs::rename(&extracted, &runtime) { + Ok(()) => {} + Err(_) if validate_supervisor_runtime(&runtime).is_ok() => {} + Err(error) => { + return Err(Error::config(format!( + "install trusted supervisor runtime '{}': {error}", + runtime.display() + ))); + } + } + validate_supervisor_runtime(&runtime) +} + async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { let mut stream = docker.create_image( Some(CreateImageOptions { @@ -3984,6 +4938,19 @@ async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { /// binary as a tar archive, and return the untarred file bytes. The /// container is always removed, even on error paths. async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { + extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_BINARY_PATH, true).await +} + +async fn extract_supervisor_runtime_archive(docker: &Docker, image: &str) -> CoreResult> { + extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_RUNTIME_PATH, false).await +} + +async fn extract_supervisor_path_archive( + docker: &Docker, + image: &str, + path: &str, + extract_single_file: bool, +) -> CoreResult> { let container_name = temp_extract_container_name(); docker .create_container( @@ -4007,7 +4974,8 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe })?; // Always tear down the extractor container, even if extraction fails. - let result = download_binary_from_container(docker, &container_name).await; + let result = + download_path_from_container(docker, &container_name, path, extract_single_file).await; if let Err(remove_err) = docker .remove_container( &container_name, @@ -4024,12 +4992,14 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe result } -async fn download_binary_from_container( +async fn download_path_from_container( docker: &Docker, container_name: &str, + path: &str, + extract_single_file: bool, ) -> CoreResult> { let options = DownloadFromContainerOptionsBuilder::default() - .path(SUPERVISOR_IMAGE_BINARY_PATH) + .path(path) .build(); let mut stream = docker.download_from_container(container_name, Some(options)); @@ -4043,11 +5013,15 @@ async fn download_binary_from_container( tar_bytes.extend_from_slice(&chunk); } - extract_first_tar_entry(&tar_bytes).map_err(|err| { - Error::config(format!( - "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", - )) - }) + if extract_single_file { + extract_first_tar_entry(&tar_bytes).map_err(|err| { + Error::config(format!( + "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", + )) + }) + } else { + Ok(tar_bytes) + } } fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 07fb2b0ffb..bdfe961e2f 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -97,14 +97,11 @@ fn runtime_config() -> DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), image_pull_policy: String::new(), sandbox_namespace: "default".to_string(), - grpc_endpoint: "https://localhost:8443".to_string(), - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), gateway_route: DockerGatewayRoute::Bridge { bind_address: SocketAddr::new( IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), DEFAULT_SERVER_PORT, ), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), }, gateway_callback_bind_address: Some(SocketAddr::new( IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), @@ -114,6 +111,11 @@ fn runtime_config() -> DockerDriverRuntimeConfig { stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: "info".to_string(), supervisor_bin: PathBuf::from("/tmp/openshell-sandbox"), + supervisor_runtime: PathBuf::from("/tmp/openshell-runtime"), + control_bin: PathBuf::from("/tmp/openshell-sandbox"), + boundary_runtime_root: PathBuf::from("/tmp/openshell-test-docker-boundaries"), + host_grpc_endpoint: "https://localhost:8443".to_string(), + gateway_tls_server_name: None, guest_tls: Some(DockerGuestTlsPaths { ca: PathBuf::from("/tmp/ca.crt"), cert: PathBuf::from("/tmp/tls.crt"), @@ -165,9 +167,189 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr allow_all_default_gpu, )), lifecycle_event_fences: DockerLifecycleEventFences::default(), + control_processes: Arc::new(tokio::sync::Mutex::new(HashMap::new())), + control_failures: Arc::new(tokio::sync::Mutex::new(HashMap::new())), } } +#[test] +fn docker_boundary_socket_path_is_short_and_namespaced() { + let sandbox = test_sandbox(); + let config = runtime_config(); + let directory = docker_boundary_mount_dir(&sandbox, &config); + let socket = directory.join("control.sock"); + let ssh_socket = docker_control_ssh_socket(&sandbox, &config); + + assert!(directory.starts_with(&config.boundary_runtime_root)); + assert!(socket.as_os_str().len() < 108); + assert!(ssh_socket.starts_with(&config.boundary_runtime_root)); + assert!(ssh_socket.as_os_str().len() < 108); + + let mut other_namespace = runtime_config(); + other_namespace.sandbox_namespace = "other".to_string(); + assert_ne!( + directory, + docker_boundary_mount_dir(&sandbox, &other_namespace) + ); + assert_ne!( + ssh_socket, + docker_control_ssh_socket(&sandbox, &other_namespace) + ); +} + +#[cfg(unix)] +#[test] +fn docker_boundary_runtime_creates_private_directory() { + use std::os::unix::fs::MetadataExt as _; + + let temporary = TempDir::new().expect("create temporary directory"); + let boundary = temporary.path().join("boundary"); + + ensure_owned_directory(&boundary, true).expect("create private boundary directory"); + + let metadata = fs::symlink_metadata(&boundary).expect("inspect boundary directory"); + assert_eq!(metadata.mode() & 0o777, 0o700); + assert_eq!(metadata.uid(), rustix::process::geteuid().as_raw()); +} + +#[cfg(unix)] +#[test] +fn docker_boundary_runtime_rejects_preexisting_symlink() { + use std::os::unix::fs::symlink; + + let temporary = TempDir::new().expect("create temporary directory"); + let target = temporary.path().join("target"); + fs::create_dir(&target).expect("create target"); + let boundary = temporary.path().join("boundary"); + symlink(&target, &boundary).expect("create boundary symlink"); + + let error = ensure_owned_directory(&boundary, true).expect_err("symlink must be rejected"); + assert!(error.contains("must not be a symlink"), "{error}"); +} + +#[cfg(unix)] +#[test] +fn docker_boundary_runtime_rejects_permissive_existing_directory() { + use std::os::unix::fs::PermissionsExt as _; + + let temporary = TempDir::new().expect("create temporary directory"); + let boundary = temporary.path().join("boundary"); + fs::create_dir(&boundary).expect("create boundary directory"); + fs::set_permissions(&boundary, fs::Permissions::from_mode(0o755)).expect("set permissive mode"); + + let error = + ensure_owned_directory(&boundary, true).expect_err("permissive directory must be rejected"); + assert!(error.contains("unsafe mode 0755"), "{error}"); +} + +#[cfg(unix)] +#[test] +fn docker_boundary_runtime_rejects_non_socket_stale_entry() { + let temporary = TempDir::new().expect("create temporary directory"); + let socket = temporary.path().join("control.sock"); + fs::write(&socket, b"not a socket").expect("write stale entry"); + + let error = remove_owned_stale_socket(&socket, 0o666, true) + .expect_err("non-socket stale entry must be rejected"); + assert!( + error.contains("not an expected owned mode-0666 Unix socket"), + "{error}" + ); +} + +#[cfg(unix)] +#[test] +fn docker_boundary_restart_removes_stale_control_and_ssh_sockets() { + use std::os::unix::fs::PermissionsExt as _; + use std::os::unix::net::UnixListener; + + let temporary = TempDir::new().expect("create temporary directory"); + let mut config = runtime_config(); + config.boundary_runtime_root = temporary.path().join("runtime"); + fs::create_dir(&config.boundary_runtime_root).expect("create runtime root"); + fs::set_permissions( + &config.boundary_runtime_root, + fs::Permissions::from_mode(0o700), + ) + .expect("restrict runtime root"); + let boundary = docker_boundary_mount_dir_by_id("sbx-123", &config); + fs::create_dir(&boundary).expect("create boundary directory"); + fs::set_permissions(&boundary, fs::Permissions::from_mode(0o700)) + .expect("restrict boundary directory"); + + let control = boundary.join("control.sock"); + let control_listener = UnixListener::bind(&control).expect("bind stale control socket"); + fs::set_permissions(&control, fs::Permissions::from_mode(0o666)).expect("set control mode"); + let ssh = docker_control_ssh_socket_by_id("sbx-123", &config); + let ssh_listener = UnixListener::bind(&ssh).expect("bind stale SSH socket"); + fs::set_permissions(&ssh, fs::Permissions::from_mode(0o600)).expect("set SSH mode"); + drop((control_listener, ssh_listener)); + + prepare_docker_boundary_restart_sockets("sbx-123", &config) + .expect("remove validated stale sockets"); + + assert!(!control.exists()); + assert!(!ssh.exists()); +} + +#[tokio::test] +async fn control_failure_overrides_running_container_readiness() { + let driver = test_driver_with_config(runtime_config()); + driver.control_failures.lock().await.insert( + "sbx-123".to_string(), + "control exited unexpectedly".to_string(), + ); + let mut sandbox = pending_sandbox_snapshot( + &test_sandbox(), + "default", + DriverCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: "BackendReady".to_string(), + message: "Container is running".to_string(), + last_transition_time: String::new(), + }, + false, + ); + + driver.apply_control_failure(&mut sandbox).await; + + let ready = sandbox + .status + .unwrap() + .conditions + .into_iter() + .find(|condition| condition.r#type == "Ready") + .expect("ready condition"); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, "ControlSupervisorExited"); + assert!(ready.message.contains("control exited unexpectedly")); +} + +#[cfg(unix)] +#[test] +fn control_command_does_not_inherit_gateway_environment() { + let _guard = ENV_LOCK.lock().unwrap(); + temp_env::with_var( + "OPENSHELL_TEST_GATEWAY_SECRET", + Some("must-not-leak"), + || { + let runtime = tokio::runtime::Runtime::new().expect("create test runtime"); + let output = runtime + .block_on(async { + new_control_command(Path::new("/usr/bin/env")) + .output() + .await + }) + .expect("run environment probe"); + assert!(output.status.success()); + assert!( + !String::from_utf8_lossy(&output.stdout).contains("OPENSHELL_TEST_GATEWAY_SECRET") + ); + }, + ); +} + #[tokio::test] async fn tracing_in_process_service_preserves_the_driver_rpc_server_boundary() { use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; @@ -682,34 +864,6 @@ async fn host_gateway_route_reports_ipv4_loopback_callback_listener() { ); } -#[test] -fn container_visible_endpoint_rewrites_loopback_hosts() { - assert_eq!( - docker_container_openshell_endpoint( - "https://localhost:8443", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "https://host.openshell.internal:17670/" - ); - assert_eq!( - docker_container_openshell_endpoint( - "http://127.0.0.1:8080", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "http://host.openshell.internal:17670/" - ); - assert_eq!( - docker_container_openshell_endpoint( - "https://gateway.internal:8443", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "https://host.openshell.internal:17670/" - ); -} - #[test] fn docker_bridge_gateway_ip_requires_ipv4_gateway() { let network = bollard::models::NetworkInspect { @@ -774,13 +928,25 @@ fn docker_gateway_route_uses_host_gateway_for_docker_desktop() { ), DockerGatewayRoute::HostGateway ); - assert_eq!( - docker_extra_hosts(&DockerGatewayRoute::HostGateway), - vec![ - "host.docker.internal:host-gateway".to_string(), - "host.openshell.internal:host-gateway".to_string() - ] - ); +} + +#[test] +fn vm_backed_docker_daemon_rejects_shared_unix_boundary_transport() { + let desktop = SystemInfo { + operating_system: Some("Docker Desktop".to_string()), + ..Default::default() + }; + let native = SystemInfo { + operating_system: Some("Ubuntu 24.04".to_string()), + ..Default::default() + }; + + let error = validate_shared_unix_boundary_transport(&desktop, false) + .expect_err("VM-backed daemon must fail before provisioning"); + assert!(error.to_string().contains("shares host Unix sockets")); + validate_shared_unix_boundary_transport(&native, false) + .expect("local native daemon supports bind-mounted Unix sockets"); + assert!(validate_shared_unix_boundary_transport(&native, true).is_err()); } #[test] @@ -825,13 +991,6 @@ fn docker_gateway_route_uses_host_gateway_for_colima() { ), DockerGatewayRoute::HostGateway ); - assert_eq!( - docker_extra_hosts(&DockerGatewayRoute::HostGateway), - vec![ - "host.docker.internal:host-gateway".to_string(), - "host.openshell.internal:host-gateway".to_string() - ] - ); } #[test] @@ -916,16 +1075,8 @@ fn docker_gateway_route_uses_bridge_gateway_for_linux_docker() { route, DockerGatewayRoute::Bridge { bind_address: "172.18.0.1:17670".parse().unwrap(), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), } ); - assert_eq!( - docker_extra_hosts(&route), - vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ] - ); } #[test] @@ -965,16 +1116,8 @@ fn docker_gateway_route_prefers_configured_host_gateway_ip() { route, DockerGatewayRoute::Bridge { bind_address: "172.20.0.4:17670".parse().unwrap(), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4)), } ); - assert_eq!( - docker_extra_hosts(&route), - vec![ - "host.docker.internal:172.20.0.4".to_string(), - "host.openshell.internal:172.20.0.4".to_string() - ] - ); } #[test] @@ -1071,89 +1214,44 @@ fn container_create_body_sets_driver_owned_pids_limit() { } #[test] -fn build_environment_sets_docker_tls_paths() { - let env = build_environment(&test_sandbox(), &runtime_config()); - assert!(env.contains(&format!("OPENSHELL_TLS_CA={TLS_CA_MOUNT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_CERT={TLS_CERT_MOUNT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_KEY={TLS_KEY_MOUNT_PATH}"))); - assert!(env.contains(&"TEMPLATE_ENV=template".to_string())); - assert!(env.contains(&"SPEC_ENV=spec".to_string())); - assert!(env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - let encoded = env - .iter() - .find_map(|entry| { - entry - .strip_prefix("OPENSHELL_MAIN_PROCESS_SPEC=") - .map(str::to_string) - }) - .expect("main-process transport"); - let main = openshell_core::sandbox_env::MainProcessConfig::decode(&encoded).unwrap(); - assert_eq!(main.command, vec!["/bin/bash", "-l"]); - assert!(main.tty); -} - -#[test] -fn build_environment_keeps_network_capabilities_driver_controlled() { - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - "spoofed".to_string(), - ); - let env = build_environment(&sandbox, &runtime_config()); - assert!(env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); -} - -#[test] -fn build_environment_protects_oci_identity_metadata() { +fn docker_child_environment_strips_supervisor_control_keys() { let mut sandbox = test_sandbox(); let spec = sandbox.spec.as_mut().unwrap(); - for (key, value) in [ - (openshell_core::sandbox_env::OCI_IMAGE_USER, "spoofed"), - (openshell_core::sandbox_env::SANDBOX_UID, "9999"), - (openshell_core::sandbox_env::SANDBOX_GID, "9999"), + for key in [ + openshell_core::sandbox_env::ENDPOINT, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, ] { - spec.environment.insert(key.to_string(), value.to_string()); + spec.environment + .insert(key.to_string(), "spoofed".to_string()); } + spec.environment + .insert("PATH".to_string(), "/agent/bin".to_string()); - let env = build_environment_for_oci_user(&sandbox, &runtime_config(), "app:staff"); + let env = docker_child_environment(&sandbox); - assert!(env.contains(&format!( - "{}=app:staff", - openshell_core::sandbox_env::OCI_IMAGE_USER - ))); - assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_UID))); - assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_GID))); - assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); - assert!(!env.iter().any(|entry| entry.ends_with("=9999"))); + assert_eq!(env.get("PATH").map(String::as_str), Some("/agent/bin")); + assert!(env.contains_key("TEMPLATE_ENV")); + assert!(env.contains_key("SPEC_ENV")); + assert!(!env.values().any(|value| value == "spoofed")); } #[test] -fn build_environment_strips_gateway_tls_server_name() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.environment.insert( - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME.to_string(), - "evil.attacker.example.com".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); +fn boundary_environment_contains_only_driver_owned_values() { + let env = build_boundary_environment(&test_sandbox(), &runtime_config()); + assert_eq!(env.len(), 2); assert!( - !env.iter().any(|entry| entry.starts_with(&format!( - "{}=", - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME - ))), - "GATEWAY_TLS_SERVER_NAME must be stripped from the supervisor environment" + env.iter() + .any(|entry| entry.starts_with("OPENSHELL_LOG_LEVEL=")) ); + assert!(env.iter().any(|entry| entry.starts_with(&format!( + "{}=", + openshell_core::sandbox_env::TELEMETRY_ENABLED + )))); } #[test] @@ -1177,14 +1275,52 @@ fn container_creation_uses_inspected_immutable_image() { assert_eq!(body.image.as_deref(), Some("sha256:immutable")); assert_eq!(body.user.as_deref(), Some("0")); assert_eq!(body.working_dir.as_deref(), Some("/")); + assert_eq!( + body.labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_TOPOLOGY)) + .map(String::as_str), + Some(LABEL_ISOLATION_TOPOLOGY_BOUNDARY_V1) + ); assert_eq!( body.cmd.as_deref(), - Some(&["--workdir".to_string(), "/workspace/project".to_string()][..]) + Some( + &[ + "--mode=boundary".to_string(), + "--boundary-config".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ][..] + ) ); - assert!(body.env.unwrap().contains(&format!( - "{}=1234:1235", - openshell_core::sandbox_env::OCI_IMAGE_USER - ))); + assert!(body.env.unwrap().iter().all(|entry| { + !entry.starts_with(&format!("{}=", openshell_core::sandbox_env::OCI_IMAGE_USER)) + })); +} + +#[test] +fn boundary_control_requirement_recognizes_marker_and_existing_command() { + let labeled = ContainerSummary { + labels: Some(HashMap::from([( + LABEL_ISOLATION_TOPOLOGY.to_string(), + LABEL_ISOLATION_TOPOLOGY_BOUNDARY_V1.to_string(), + )])), + ..Default::default() + }; + let pre_marker = ContainerSummary { + command: Some( + "/usr/local/bin/openshell-sandbox --mode=boundary --boundary-config /run/config.json" + .to_string(), + ), + ..Default::default() + }; + let legacy = ContainerSummary { + command: Some("/usr/local/bin/openshell-sandbox".to_string()), + ..Default::default() + }; + + assert!(container_requires_boundary_control(&labeled)); + assert!(container_requires_boundary_control(&pre_marker)); + assert!(!container_requires_boundary_control(&legacy)); } #[test] @@ -1354,78 +1490,19 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( } #[test] -fn build_environment_keeps_path_driver_controlled() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.environment - .insert("PATH".to_string(), "/malicious/spec/bin".to_string()); - spec.template - .as_mut() - .unwrap() - .environment - .insert("PATH".to_string(), "/malicious/template/bin".to_string()); - - let env = build_environment(&sandbox, &runtime_config()); - let path_entries = env - .iter() - .filter(|entry| entry.starts_with("PATH=")) - .collect::>(); - - let expected_path = format!("PATH={SUPERVISOR_PATH}"); - assert_eq!(path_entries.len(), 1); - assert_eq!(path_entries[0], &expected_path); -} - -#[test] -fn build_environment_keeps_telemetry_toggle_driver_controlled() { - let _guard = ENV_LOCK.lock().unwrap(); - temp_env::with_vars( - [( - openshell_core::sandbox_env::TELEMETRY_ENABLED, - Some("false"), - )], - || { - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - "true".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); - let telemetry_entries = env - .iter() - .filter(|entry| { - entry.starts_with(&format!( - "{}=", - openshell_core::sandbox_env::TELEMETRY_ENABLED - )) - }) - .collect::>(); - - assert_eq!(telemetry_entries.len(), 1); - assert_eq!( - telemetry_entries[0], - &format!("{}=false", openshell_core::sandbox_env::TELEMETRY_ENABLED) - ); - }, - ); -} - -#[test] -fn build_binds_uses_docker_tls_directory() { - let binds = build_binds(&test_sandbox(), &runtime_config()).unwrap(); +fn build_binds_exposes_only_boundary_runtime_material() { + let binds = build_binds(&test_sandbox(), &runtime_config()); let targets = binds .iter() .filter_map(|bind| bind.split(':').nth(1).map(String::from)) .collect::>(); assert!(targets.contains(&SUPERVISOR_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_CA_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_CERT_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_KEY_MOUNT_PATH.to_string())); + assert!(targets.contains(&SUPERVISOR_RUNTIME_MOUNT_PATH.to_string())); + assert!(targets.contains(&BOUNDARY_MOUNT_PATH.to_string())); assert!( - targets + !targets .iter() - .all(|target| target.starts_with(TLS_MOUNT_DIR) || target == SUPERVISOR_MOUNT_PATH) + .any(|target| target.starts_with(TLS_MOUNT_DIR)) ); } @@ -1980,27 +2057,6 @@ fn docker_nonlocal_volume_with_bind_option_is_not_bind_backed() { assert!(!docker_volume_is_bind_backed(&volume)); } -#[test] -fn build_environment_uses_token_file_without_raw_token_env() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.sandbox_token = "secret.jwt.value".to_string(); - spec.environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN.to_string(), - "user-provided-token".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); - - assert!(!env.iter().any(|entry| { - entry.starts_with(&format!("{}=", openshell_core::sandbox_env::SANDBOX_TOKEN)) - })); - assert!(env.contains(&format!( - "{}={SANDBOX_TOKEN_MOUNT_PATH}", - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE - ))); -} - #[test] fn managed_container_label_filters_include_gateway_namespace() { let filters = @@ -2013,7 +2069,7 @@ fn managed_container_label_filters_include_gateway_namespace() { } #[test] -fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { +fn build_container_create_body_replaces_inherited_cmd_with_boundary_mode() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); assert_eq!( @@ -2022,7 +2078,11 @@ fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { ); assert_eq!( create_body.cmd, - Some(vec!["--workdir".to_string(), "/sandbox".to_string()]) + Some(vec![ + "--mode=boundary".to_string(), + "--boundary-config".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ]) ); assert_eq!( create_body @@ -2040,25 +2100,16 @@ fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { host_config.security_opt.as_ref(), Some(&vec!["apparmor=unconfined".to_string()]) ); + assert_eq!(host_config.network_mode.as_deref(), Some("none")); assert_eq!( - host_config.network_mode.as_deref(), - Some(DEFAULT_DOCKER_NETWORK_NAME) - ); - assert_eq!( - host_config.extra_hosts.as_ref(), - Some(&vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ]) - ); - assert_eq!( - create_body - .networking_config - .as_ref() - .and_then(|config| config.endpoints_config.as_ref()) - .and_then(|endpoints| endpoints.get(DEFAULT_DOCKER_NETWORK_NAME)), - Some(&EndpointSettings::default()) + host_config.extra_hosts, + Some(vec![ + "host.openshell.internal:host-gateway".to_string(), + "host.docker.internal:host-gateway".to_string(), + ]), + "networkless workloads still receive stable host aliases" ); + assert!(create_body.networking_config.is_none()); } #[test] @@ -2507,22 +2558,22 @@ fn require_sandbox_identifier_rejects_when_id_and_name_are_empty() { } #[test] -fn build_container_create_body_uses_bridge_network() { +fn build_container_create_body_disables_docker_networking() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = create_body.host_config.expect("host_config is populated"); assert_eq!( host_config.network_mode, - Some(DEFAULT_DOCKER_NETWORK_NAME.to_string()), - "sandbox should join the driver-managed bridge network" + Some("none".to_string()), + "only the nested workload namespace should receive mediated egress" ); assert_eq!( host_config.extra_hosts, Some(vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() + "host.openshell.internal:host-gateway".to_string(), + "host.docker.internal:host-gateway".to_string(), ]), - "sandbox should expose stable host aliases for gateway callbacks" + "networkless workloads still need stable host aliases in /etc/hosts" ); } @@ -2758,15 +2809,21 @@ fn docker_guest_tls_paths_require_all_files_for_https() { fn linux_supervisor_candidates_follow_daemon_arch() { assert_eq!( linux_supervisor_candidates("amd64"), - vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )] + vec![ + docker_workspace_root() + .join("target/x86_64-unknown-linux-gnu/release/openshell-sandbox") + ] ); assert_eq!( linux_supervisor_candidates("arm64"), - vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )] + vec![ + docker_workspace_root() + .join("target/aarch64-unknown-linux-gnu/release/openshell-sandbox") + ] + ); + assert!( + linux_supervisor_candidates("amd64")[0].is_absolute(), + "developer fallback must never resolve relative to the gateway cwd" ); } diff --git a/e2e/rust/tests/forward_proxy_l7_bypass.rs b/e2e/rust/tests/forward_proxy_l7_bypass.rs index f5df4f53e3..29261ff635 100644 --- a/e2e/rust/tests/forward_proxy_l7_bypass.rs +++ b/e2e/rust/tests/forward_proxy_l7_bypass.rs @@ -10,13 +10,13 @@ use std::io::Write; -use openshell_e2e::harness::container::ContainerHttpServer; +use openshell_e2e::harness::container::HostSupportContainer; use openshell_e2e::harness::sandbox::SandboxGuard; use tempfile::NamedTempFile; -const TEST_SERVER_ALIAS: &str = "rest-l7.openshell.test"; +const TEST_SERVER_HOST: &str = "host.openshell.internal"; -async fn start_test_server() -> Result { +async fn start_test_server() -> Result { let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer class Handler(BaseHTTPRequestHandler): @@ -34,7 +34,7 @@ class Handler(BaseHTTPRequestHandler): HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() "#; - ContainerHttpServer::start_python(TEST_SERVER_ALIAS, script).await + HostSupportContainer::start_python(script, 8000).await } fn write_policy_with_l7_rules(host: &str, port: u16) -> Result { @@ -100,7 +100,7 @@ network_policies: async fn forward_proxy_allows_l7_permitted_request() { let server = start_test_server().await.expect("start test server"); let policy = - write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); + write_policy_with_l7_rules(TEST_SERVER_HOST, server.port).expect("write custom policy"); let policy_path = policy .path() .to_str() @@ -129,7 +129,7 @@ for attempt in range(6): break print(json.dumps(last)) "#, - host = server.host, + host = TEST_SERVER_HOST, port = server.port, ); @@ -150,7 +150,7 @@ print(json.dumps(last)) async fn forward_proxy_denies_l7_blocked_request() { let server = start_test_server().await.expect("start test server"); let policy = - write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); + write_policy_with_l7_rules(TEST_SERVER_HOST, server.port).expect("write custom policy"); let policy_path = policy .path() .to_str() @@ -170,7 +170,7 @@ except urllib.error.HTTPError as e: except Exception as e: print(json.dumps({{"status": -1, "error": str(e)}})) "#, - host = server.host, + host = TEST_SERVER_HOST, port = server.port, );