From 273eba4b2d4f17707e2802b3ee934005c51dcfd3 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 2 Sep 2026 17:50:42 -0700 Subject: [PATCH 1/6] feat(docker): adopt shared supervisor boundary mode Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/Cargo.toml | 7 +- crates/openshell-driver-docker/README.md | 84 +- .../openshell-driver-docker/src/isolation.rs | 107 ++ crates/openshell-driver-docker/src/lib.rs | 913 ++++++++++++++++-- crates/openshell-driver-docker/src/tests.rs | 104 +- 5 files changed, 1031 insertions(+), 184 deletions(-) create mode 100644 crates/openshell-driver-docker/src/isolation.rs diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 7a5fa3fe3d..b66a7f3274 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-otel = { path = "../openshell-otel" } opentelemetry = { workspace = true } @@ -38,15 +39,17 @@ miette = { workspace = true } toml = { workspace = true } tower-http = { workspace = true } http = { workspace = true } +rand = { workspace = true } +sha2 = { workspace = true } +tar = "0.4" +tempfile = "3" [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } prost-types = { workspace = true } -tar = "0.4" temp-env = "0.3" -tempfile = "3" tracing-subscriber = { workspace = true } [lints] diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index bbd7e69b88..bb96fd5b88 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -29,18 +29,31 @@ no candidate responds. ## Runtime Model -The gateway runs as a host process. The Docker driver creates one container per -sandbox and starts the `openshell-sandbox` supervisor inside that container. The -supervisor then creates the nested sandbox namespace for the agent process. +The gateway and Docker driver run on the host. For each sandbox, the driver +starts `openshell-sandbox --mode=control` beside the gateway and +`openshell-sandbox --mode=boundary` as PID 1 in the workload container. The two +roles use the shared RFC 0012 protocol over a private bind-mounted Unix socket. + +Control owns the gateway session, policy, SSH relay, and network proxy. The +boundary owns the process tree, exec/signal/wait/PTY operations, loopback port +forwarding, procfs binary identity, and the workload network namespace. The +boundary receives no gateway JWT, client TLS private key, or policy authority. + +The driver bind-mounts the boundary socket from a short, digest-keyed runtime +directory under `/tmp/openshell-docker-boundaries`. This keeps the host endpoint +below Linux's Unix-socket path limit even when the gateway state directory is +deeply nested. The full protected topology and gateway material remain in the +normal driver state directory, and sandbox cleanup removes both locations. ## Stop and Start -Stop stops the managed container without removing it. Docker retains the -container writable layer, attached volumes, labels, token material, and restart -policy. Start starts that same container, so files in the resolved OCI -workspace remain available. A durably stopped sandbox is excluded from -gateway startup recovery and stays stopped across gateway restarts. Delete -continues to force-remove the container and clean up driver-owned material. +Stop terminates host control and stops the managed container without removing +it. Docker retains the container writable layer, attached volumes, labels, and +the protected driver topology. Start starts that same container and recreates +host control, so files in the resolved OCI workspace remain available. A +durably stopped sandbox is excluded from gateway startup recovery and stays +stopped across gateway restarts. Delete force-removes the container and the +driver-owned boundary directory. Graceful gateway shutdown sends `StopSandbox` for each sandbox whose persisted phase requires running compute without changing that persisted intent. On startup, the gateway sends an idempotent `StartSandbox` request for the same @@ -50,9 +63,9 @@ remain excluded. Before creating the container, the driver inspects the final sandbox image and captures its immutable image ID, raw OCI `Config.User`, and OCI `Config.WorkingDir`. Container creation uses that image ID, preventing a -mutable tag from changing between inspection and launch. The supervisor runs as -root, resolves omitted policy identity fields from the image declaration, and -drops only agent children to the resulting identity. Named OCI components +mutable tag from changing between inspection and launch. The boundary runs as +root, resolves omitted policy identity fields from the image declaration inside +the image filesystem, and drops only agent children to the resulting identity. Named OCI components remain names after validation; a missing group is filled with the user's numeric primary GID. Explicit `process.run_as_user` and `process.run_as_group` values take precedence independently. @@ -83,14 +96,12 @@ are rejected, as are paths that overlap concrete OpenShell control resources. The workspace is the child cwd and `HOME`. The supervisor starts from `/`, then reports an invalid workdir as a readiness failure. -Docker containers join an OpenShell-managed bridge network. The driver injects -`host.openshell.internal` and `host.docker.internal` so supervisors have stable -names for reaching the gateway host. On Docker Desktop, Colima, Rancher -Desktop, OrbStack, and macOS-hosted gateways, those names use Docker's -`host-gateway` alias. The driver requests a separate IPv4 loopback callback -listener when the primary listener does not already cover it. On native Linux -Docker, the gateway also binds the bridge gateway IP so containers can call -back to the host process. +Boundary containers use Docker `network_mode=none`. The nested agent network +namespace can reach only the boundary's mediation listener. Each accepted TCP +stream and its procfs-derived executable identity cross the private Unix socket +to host control, which applies policy before dialing the destination. Host +control reaches the gateway directly; the existing managed bridge listener is +retained for compatibility with legacy containers created before this split. ## Container Contract @@ -100,13 +111,15 @@ contract: | Setting | Purpose | |---|---| | `user = "0"` | The supervisor needs root inside the container to prepare namespaces, mounts, Landlock, and seccomp. | -| `network_mode = openshell` | Places the supervisor on the managed Docker bridge network. | +| `network_mode = none` | Removes direct container egress; the nested workload namespace can emit only through the authenticated boundary stream. | | `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | | `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | | `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | | `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Set `[openshell.drivers.docker].sandbox_pids_limit = 0` to inherit the Docker/runtime default. | | CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | -| `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | +| Read-only supervisor/runtime mounts | Supplies the boundary binary and trusted `ip`/`nft` helper runtime without trusting workload-image tools. | +| Private boundary bind mount | Carries the protected boundary config and Unix control socket. The directory is host-owned mode `0700`; workload children cannot access it. | +| `policy-dns-transparent-tcp` capability | Declares that the shared boundary can own namespace-local DNS/TCP capture while host control retains authorization, pinned dialing, relaying, and OCSF decisions. The marker is not exposed to the workload environment. | The agent child process does not retain these supervisor privileges. @@ -127,8 +140,9 @@ mount types: Host bind mounts are disabled by default because they expose gateway host paths to sandbox requests. Image mounts are not part of the Docker -driver-config schema. The driver still uses internal bind mounts for -OpenShell-owned supervisor, token, and TLS material. +driver-config schema. The driver still uses internal bind mounts for the +OpenShell-owned boundary binary, trusted helper runtime, and private socket +directory. Gateway token and TLS material remain on the host. Docker `bind` mounts accept `source`, `target`, optional `read_only`, and an optional `selinux_label` of `shared` (applies `:z`) or `private` (applies @@ -154,8 +168,9 @@ openshell sandbox create \ ## Supervisor Binary Resolution -The Docker driver bind-mounts a host-side Linux `openshell-sandbox` binary into -each sandbox container. Resolution order is: +The Docker driver bind-mounts a host-side Linux `openshell-sandbox` boundary +binary and trusted helper runtime into each sandbox container. Resolution order +for the Linux artifact is: 1. `supervisor_bin` in `[openshell.drivers.docker]`. 2. `supervisor_image` in `[openshell.drivers.docker]`, extracting @@ -164,13 +179,19 @@ each sandbox container. Resolution order is: 4. A local Linux cargo target build for the Docker daemon architecture. 5. The release-matched default supervisor image, extracting `/openshell-sandbox`. +When a selected binary has no valid sibling `openshell-runtime`, the driver +extracts `/openshell-runtime` from the configured or release-matched supervisor +image. Host control uses the native `openshell-sandbox` beside the gateway (or a +local native cargo build); Linux hosts can reuse the boundary binary. + Release and Docker-image gateway builds bake the matching supervisor image tag into the binary at compile time. The default Docker supervisor image is not `:latest` unless a custom build explicitly sets that tag. ## Callback and TLS -`OPENSHELL_ENDPOINT` is injected from the gateway's configured gRPC endpoint. +`OPENSHELL_ENDPOINT` is injected only into host control from the gateway's +configured gRPC endpoint. When no endpoint is configured, the driver uses `host.openshell.internal:` with the appropriate HTTP or HTTPS scheme. Set `host_gateway_ip` only when the host has an explicit, locally @@ -178,8 +199,8 @@ assigned address that containers should use for callbacks; package-managed macOS gateways should leave it unset. For HTTPS endpoints, the server certificate must include the endpoint host as a -subject alternative name. Docker sandboxes also need the client TLS bundle -mounted into the container and exposed with: +subject alternative name. The client TLS bundle is exposed only to host control +with: - `OPENSHELL_TLS_CA` - `OPENSHELL_TLS_CERT` @@ -189,8 +210,9 @@ HTTP endpoints reject TLS material because the supervisor would not use it. ## Environment Ownership -The driver merges template environment and sandbox spec environment first, then -overwrites security-critical keys: +The driver writes template and sandbox environment into the protected boundary +config. Boundary mode exposes that map only to workload children. Host control +owns these security-critical keys: - `OPENSHELL_ENDPOINT` - `OPENSHELL_SANDBOX_ID` diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs new file mode 100644 index 0000000000..a1011e8793 --- /dev/null +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Docker provisioning for the shared authenticated boundary protocol. +//! +//! Docker owns only the container/socket topology and immutable OCI resource +//! claims. Lifecycle, process, network, identity, and wire behavior live in +//! `openshell-isolation-interface` and `openshell-sandbox`. + +use std::collections::{BTreeMap, HashMap}; +use std::net::IpAddr; +use std::path::PathBuf; + +use openshell_isolation_interface::boundary_protocol::{ + BOUNDARY_PROTOCOL_VERSION, BoundaryAgentIdentity, BoundaryConfig, BoundaryListener, + BoundaryTopology, BoundaryTransport, +}; + +/// Driver-owned inputs that bind one Docker container to one boundary. +pub struct DockerBoundarySpec { + pub boundary_id: String, + pub bootstrap_token: String, + pub container_id: String, + pub image_identity: String, + pub listener_socket: PathBuf, + pub control_socket: PathBuf, + pub host_gateway_ip: Option, + pub oci_user: String, + pub trusted_runtime_root: PathBuf, + pub child_env: HashMap, +} + +/// Protected container config and matching host descriptor. +pub struct DockerBoundaryProvisioning { + pub boundary_config: BoundaryConfig, + pub topology: BoundaryTopology, +} + +impl DockerBoundarySpec { + /// Produce both sides of the common protocol from the same immutable + /// Docker coordinates so attach cannot bind a different container. + #[must_use] + pub fn provision(self) -> DockerBoundaryProvisioning { + let resource_claims = BTreeMap::from([ + ("docker.container_id".to_string(), self.container_id), + ("docker.image_identity".to_string(), self.image_identity), + ]); + DockerBoundaryProvisioning { + boundary_config: BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: self.boundary_id.clone(), + bootstrap_token: self.bootstrap_token.clone(), + listener: BoundaryListener::Unix { + socket_path: self.listener_socket, + }, + resource_claims: resource_claims.clone(), + agent_identity: BoundaryAgentIdentity::OciUser { + declaration: self.oci_user, + }, + protect_config_file: true, + trusted_runtime_root: self.trusted_runtime_root, + child_env: self.child_env, + }, + topology: BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: self.boundary_id, + transport: BoundaryTransport::Unix { + socket_path: self.control_socket, + }, + host_gateway_ip: self.host_gateway_ip, + resource_claims, + bootstrap_token: self.bootstrap_token, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provisioning_binds_container_and_image_claims() { + let provisioned = DockerBoundarySpec { + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "a".repeat(64), + container_id: "sha256:container".to_string(), + image_identity: "sha256:image".to_string(), + listener_socket: PathBuf::from("/run/openshell/boundary/control.sock"), + control_socket: PathBuf::from("/host/control.sock"), + host_gateway_ip: Some(IpAddr::from([127, 0, 0, 1])), + oci_user: "1000:1000".to_string(), + trusted_runtime_root: PathBuf::from("/opt/openshell/bin/openshell-runtime"), + child_env: HashMap::new(), + } + .provision(); + + assert_eq!( + provisioned.boundary_config.resource_claims, + provisioned.topology.resource_claims + ); + assert_eq!( + provisioned.topology.resource_claims["docker.container_id"], + "sha256:container" + ); + } +} diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d599cac169..981051f16d 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5,15 +5,16 @@ #![allow(clippy::result_large_err)] +mod isolation; pub mod otel_tracing; use bollard::Docker; use bollard::errors::Error as BollardError; use bollard::models::{ ContainerCreateBody, ContainerState, ContainerStateStatusEnum, ContainerSummary, - ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, EndpointSettings, HostConfig, Mount, - MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, NetworkCreateRequest, NetworkingConfig, - ProgressDetail, SystemInfo, + ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, HostConfig, Mount, + MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, NetworkCreateRequest, ProgressDetail, + SystemInfo, }; use bollard::query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, @@ -55,16 +56,21 @@ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; use openshell_core::{Error, Result as CoreResult}; +use openshell_isolation_interface::boundary_protocol::BoundaryTopology; use opentelemetry::trace::TraceContextExt as _; +use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::pin::Pin; +use std::process::Stdio; use std::sync::Arc; use std::task::{Context, Poll}; use std::time::Duration; -use tokio::sync::{Mutex, broadcast, mpsc}; +use tokio::process::Command; +use tokio::sync::{Mutex, broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; @@ -77,10 +83,25 @@ const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; +const SUPERVISOR_RUNTIME_MOUNT_PATH: &str = "/opt/openshell/bin/openshell-runtime"; +const SUPERVISOR_IMAGE_RUNTIME_PATH: &str = "/openshell-runtime"; +const BOUNDARY_MOUNT_PATH: &str = "/run/openshell/boundary"; +const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/run/openshell/boundary/config.json"; +const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/run/openshell/boundary/control.sock"; +const DOCKER_BOUNDARY_RUNTIME_ROOT: &str = "/tmp/openshell-docker-boundaries"; +const DRIVER_ADMITTED_BACKEND: &str = "docker"; +const TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload"; +const MAIN_PROCESS_SPEC_FILE: &str = "main-process.json"; +const WORKSPACE_ROOT_FILE: &str = "workspace-root"; +#[cfg(test)] const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; +#[cfg(test)] const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; +#[cfg(test)] const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; +#[cfg(test)] const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; +#[cfg(test)] const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; @@ -200,14 +221,18 @@ struct DockerDriverRuntimeConfig { default_image: String, image_pull_policy: String, sandbox_namespace: String, + #[cfg(test)] grpc_endpoint: String, - network_name: String, gateway_route: DockerGatewayRoute, gateway_callback_bind_address: Option, ssh_socket_path: String, stop_timeout_secs: u32, log_level: String, supervisor_bin: PathBuf, + supervisor_runtime: PathBuf, + control_bin: PathBuf, + host_grpc_endpoint: String, + gateway_tls_server_name: Option, guest_tls: Option, daemon_version: String, supports_gpu: bool, @@ -233,6 +258,12 @@ pub struct DockerComputeDriver { pending: Arc>>, gpu_selector: Arc, lifecycle_event_fences: DockerLifecycleEventFences, + control_processes: Arc>>, +} + +struct DockerControlProcess { + shutdown: Option>, + task: JoinHandle<()>, } /// Per-sandbox container exit timestamps that fence snapshots from an earlier run. @@ -613,6 +644,28 @@ impl DockerComputeDriver { docker_config.grpc_endpoint = format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); } + let host_grpc_endpoint = + docker_host_openshell_endpoint(&docker_config.grpc_endpoint, &gateway_route)?; + let original_gateway_url = Url::parse(&docker_config.grpc_endpoint).map_err(|error| { + Error::config(format!( + "invalid docker grpc_endpoint '{}': {error}", + docker_config.grpc_endpoint + )) + })?; + let host_gateway_url = Url::parse(&host_grpc_endpoint).map_err(|error| { + Error::config(format!( + "invalid normalized Docker host grpc_endpoint '{host_grpc_endpoint}': {error}" + )) + })?; + let gateway_tls_server_name = (original_gateway_url.scheme() == "https" + && original_gateway_url.host_str() != host_gateway_url.host_str()) + .then(|| { + original_gateway_url + .host_str() + .unwrap_or_default() + .to_string() + }); + #[cfg(test)] let grpc_endpoint = docker_container_openshell_endpoint( &docker_config.grpc_endpoint, HOST_OPENSHELL_INTERNAL, @@ -620,6 +673,9 @@ impl DockerComputeDriver { ); let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; + let supervisor_runtime = + resolve_supervisor_runtime(&docker, &docker_config, &supervisor_bin).await?; + let control_bin = resolve_control_supervisor_bin(&supervisor_bin)?; let guest_tls = docker_guest_tls_paths(&docker_config)?; let driver = Self { @@ -628,14 +684,18 @@ impl DockerComputeDriver { default_image: docker_config.default_image.clone(), image_pull_policy: docker_config.image_pull_policy.clone(), sandbox_namespace: docker_config.sandbox_namespace.clone(), + #[cfg(test)] grpc_endpoint, - network_name, gateway_route, gateway_callback_bind_address, ssh_socket_path: docker_config.ssh_socket_path.clone(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: gateway_log_level.to_string(), supervisor_bin, + supervisor_runtime, + control_bin, + host_grpc_endpoint, + gateway_tls_server_name, guest_tls, daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), supports_gpu, @@ -650,6 +710,7 @@ impl DockerComputeDriver { allow_all_default_gpu, )), lifecycle_event_fences: DockerLifecycleEventFences::default(), + control_processes: Arc::new(Mutex::new(HashMap::new())), }; let poll_driver = driver.clone(); @@ -1007,11 +1068,23 @@ impl DockerComputeDriver { image.ref = %template.image, )) .await?; + prepare_docker_boundary_state_dir(sandbox, &self.config) + .await + .map_err(|status| { + DockerProvisioningFailure::new("BoundaryStateCreateFailed", status.message()) + })?; let token_file_created = write_sandbox_token_file(sandbox, &self.config) .await .map_err(|status| { DockerProvisioningFailure::new("SandboxTokenWriteFailed", status.message()) })?; + if !token_file_created { + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "SandboxTokenWriteFailed", + "Docker control mode requires a gateway sandbox token", + )); + } let container_name = container_name_for_sandbox(sandbox); let gpu_devices = self @@ -1022,9 +1095,7 @@ impl DockerComputeDriver { ) .await .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } + cleanup_docker_boundary_state(sandbox, &self.config); DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; let create_body = build_container_create_body_for_image( @@ -1035,12 +1106,10 @@ impl DockerComputeDriver { &image, ) .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } + cleanup_docker_boundary_state(sandbox, &self.config); DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; - async { + let created = async { openshell_otel::record_error_result( self.docker .create_container( @@ -1053,9 +1122,7 @@ impl DockerComputeDriver { ) .await .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } + cleanup_docker_boundary_state(sandbox, &self.config); DockerProvisioningFailure::from_status( "ContainerCreateFailed", create_status_from_docker_error("create docker sandbox container", err), @@ -1078,6 +1145,25 @@ impl DockerComputeDriver { HashMap::from([("container_name".to_string(), container_name.clone())]), ); + let topology = + match prepare_docker_boundary_files(sandbox, &self.config, &created.id, &image).await { + Ok(topology) => topology, + Err(status) => { + let _ = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "BoundaryConfigWriteFailed", + status.message(), + )); + } + }; + let start_result = async { openshell_otel::record_error_result( self.docker.start_container(&container_name, None).await, @@ -1107,14 +1193,30 @@ impl DockerComputeDriver { "Failed to clean up Docker container after start failure" ); } - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } + cleanup_docker_boundary_state(sandbox, &self.config); return Err(DockerProvisioningFailure::from_status( "ContainerStartFailed", create_status_from_docker_error("start docker sandbox container", err), )); } + let control = match spawn_docker_control_process(sandbox, &self.config, &topology).await { + Ok(control) => control, + Err(status) => { + let _ = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "ControlSupervisorStartFailed", + status.message(), + )); + } + }; + self.replace_control_process(&sandbox.id, control).await; self.publish_docker_progress( &sandbox.id, "Started", @@ -1135,6 +1237,54 @@ impl DockerComputeDriver { span_status.finish(Ok(())) } + async fn replace_control_process(&self, sandbox_id: &str, process: DockerControlProcess) { + let previous = self + .control_processes + .lock() + .await + .insert(sandbox_id.to_string(), process); + if let Some(previous) = previous { + stop_docker_control_process(previous).await; + } + } + + async fn stop_control_process(&self, sandbox_id: &str) { + let process = self.control_processes.lock().await.remove(sandbox_id); + if let Some(process) = process { + stop_docker_control_process(process).await; + } + } + + async fn ensure_control_process_for_container( + &self, + container: &ContainerSummary, + ) -> Result<(), Status> { + let Some(sandbox) = sandbox_from_container_summary(container) else { + return Err(Status::internal( + "managed Docker container is missing sandbox identity labels", + )); + }; + let stale = { + let mut processes = self.control_processes.lock().await; + match processes.get(&sandbox.id) { + Some(process) if !process.task.is_finished() => return Ok(()), + Some(_) => processes.remove(&sandbox.id), + None => None, + } + }; + if let Some(stale) = stale { + stop_docker_control_process(stale).await; + } + let Some(topology) = read_docker_boundary_topology(&sandbox.id, &self.config).await? else { + // A container created by the legacy combined supervisor has no + // topology file and remains self-managing across an upgrade. + return Ok(()); + }; + let process = spawn_docker_control_process(&sandbox, &self.config, &topology).await?; + self.replace_control_process(&sandbox.id, process).await; + Ok(()) + } + async fn delete_sandbox_inner( &self, sandbox_id: &str, @@ -1146,6 +1296,9 @@ impl DockerComputeDriver { { task.abort(); } + if let Some(record) = pending.as_ref() { + self.stop_control_process(&record.sandbox.id).await; + } let Some(container) = self .find_managed_container_summary(sandbox_id, sandbox_name) @@ -1162,11 +1315,11 @@ impl DockerComputeDriver { .await { Ok(()) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); + cleanup_docker_boundary_state(&record.sandbox, &self.config); return Ok(true); } Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); + cleanup_docker_boundary_state(&record.sandbox, &self.config); return Ok(true); } Err(err) => { @@ -1179,6 +1332,12 @@ impl DockerComputeDriver { let Some(target) = summary_container_target(&container) else { return Ok(pending.is_some()); }; + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + self.stop_control_process(resolved_sandbox_id).await; match self .docker @@ -1189,11 +1348,11 @@ impl DockerComputeDriver { .await { Ok(()) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + cleanup_docker_boundary_state_by_id(resolved_sandbox_id, &self.config); Ok(true) } Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + cleanup_docker_boundary_state_by_id(resolved_sandbox_id, &self.config); Ok(pending.is_some()) } Err(err) => Err(internal_status("delete docker sandbox container", err)), @@ -1206,10 +1365,11 @@ impl DockerComputeDriver { .await? else { if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { + self.stop_control_process(&record.sandbox.id).await; if let Some(task) = record.task { task.abort(); } - cleanup_sandbox_token_file(&record.sandbox, &self.config); + cleanup_docker_boundary_state(&record.sandbox, &self.config); self.publish_deleted(record.sandbox.id); return Ok(()); } @@ -1218,6 +1378,12 @@ impl DockerComputeDriver { let Some(target) = summary_container_target(&container) else { return Err(Status::not_found("sandbox container has no id or name")); }; + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + self.stop_control_process(resolved_sandbox_id).await; match self .docker @@ -1287,6 +1453,8 @@ impl DockerComputeDriver { }; let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); if !container_state_needs_start(state) { + self.ensure_control_process_for_container(&container) + .await?; return Ok(true); } @@ -1311,13 +1479,16 @@ impl DockerComputeDriver { .record_previous_exit(sandbox_id, previous_finished_at.as_deref()); match self.docker.start_container(&target, None).await { - Ok(()) => Ok(true), + Ok(()) => {} // Already running — race with another start path or the // restart policy. Treat as success. - Err(err) if is_not_modified_error(&err) => Ok(true), - Err(err) if is_not_found_error(&err) => Ok(false), - Err(err) => Err(internal_status("start docker sandbox container", err)), + Err(err) if is_not_modified_error(&err) => {} + Err(err) if is_not_found_error(&err) => return Ok(false), + Err(err) => return Err(internal_status("start docker sandbox container", err)), } + self.ensure_control_process_for_container(&container) + .await?; + Ok(true) } async fn reserve_pending_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), Status> { @@ -1386,7 +1557,7 @@ impl DockerComputeDriver { sandbox: &DriverSandbox, failure: &DockerProvisioningFailure, ) { - cleanup_sandbox_token_file(sandbox, &self.config); + cleanup_docker_boundary_state(sandbox, &self.config); let snapshot = pending_sandbox_snapshot( sandbox, &self.config.sandbox_namespace, @@ -2688,36 +2859,75 @@ fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { }) } -fn build_binds( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result, Status> { - let mut binds = vec![format!( - "{}:{}:ro,z", - config.supervisor_bin.display(), - SUPERVISOR_MOUNT_PATH - )]; - if let Some(tls) = &config.guest_tls { - binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); - binds.push(format!( +fn build_binds(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { + vec![ + format!( "{}:{}:ro,z", - tls.cert.display(), - TLS_CERT_MOUNT_PATH - )); - binds.push(format!("{}:{}:ro,z", tls.key.display(), TLS_KEY_MOUNT_PATH)); - } - if sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.is_empty()) - { - binds.push(format!( + config.supervisor_bin.display(), + SUPERVISOR_MOUNT_PATH + ), + format!( "{}:{}:ro,z", - sandbox_token_host_path(sandbox, config)?.display(), - SANDBOX_TOKEN_MOUNT_PATH - )); - } - Ok(binds) + config.supervisor_runtime.display(), + SUPERVISOR_RUNTIME_MOUNT_PATH + ), + format!( + "{}:{}:rw,z", + docker_boundary_mount_dir(sandbox, config).display(), + BOUNDARY_MOUNT_PATH + ), + ] +} + +fn docker_boundary_state_dir( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + docker_boundary_state_dir_by_id(&sandbox.id, config) +} + +fn docker_boundary_state_dir_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result { + sandbox_token_host_path_by_id(sandbox_id, config).and_then(|path| { + path.parent() + .map(Path::to_path_buf) + .ok_or_else(|| Status::internal("docker boundary state path has no parent")) + }) +} + +fn docker_boundary_mount_dir( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> PathBuf { + docker_boundary_mount_dir_by_id(&sandbox.id, config) +} + +fn docker_boundary_mount_dir_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> PathBuf { + let mut hasher = Sha256::new(); + hasher.update(config.sandbox_namespace.as_bytes()); + hasher.update([0]); + hasher.update(sandbox_id.as_bytes()); + let digest = format!("{:x}", hasher.finalize()); + Path::new(DOCKER_BOUNDARY_RUNTIME_ROOT).join(&digest[..32]) +} + +fn docker_control_ssh_socket( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> PathBuf { + docker_control_ssh_socket_by_id(&sandbox.id, config) +} + +fn docker_control_ssh_socket_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> PathBuf { + docker_boundary_mount_dir_by_id(sandbox_id, config).with_extension("ssh") } fn sandbox_token_host_path( @@ -2779,39 +2989,313 @@ async fn write_sandbox_token_file( Ok(true) } -fn cleanup_sandbox_token_file(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { - cleanup_sandbox_token_file_by_id(&sandbox.id, config); +async fn prepare_docker_boundary_state_dir( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + openshell_core::paths::create_dir_restricted(&directory).map_err(|error| { + Status::internal(format!( + "create Docker boundary state directory {}: {error}", + directory.display() + )) + })?; + let boundary_directory = docker_boundary_mount_dir(sandbox, config); + let boundary_root = boundary_directory + .parent() + .ok_or_else(|| Status::internal("Docker boundary runtime path has no parent"))?; + openshell_core::paths::create_dir_restricted(boundary_root).map_err(|error| { + Status::internal(format!( + "create Docker boundary runtime root {}: {error}", + boundary_root.display() + )) + })?; + openshell_core::paths::create_dir_restricted(&boundary_directory).map_err(|error| { + Status::internal(format!( + "create Docker boundary mount directory {}: {error}", + boundary_directory.display() + )) + })?; + let socket = boundary_directory.join("control.sock"); + if let Err(error) = tokio::fs::remove_file(&socket).await + && error.kind() != std::io::ErrorKind::NotFound + { + return Err(Status::internal(format!( + "remove stale Docker boundary socket {}: {error}", + socket.display() + ))); + } + Ok(directory) +} + +async fn write_docker_boundary_file(path: &Path, contents: &[u8]) -> Result<(), Status> { + tokio::fs::write(path, contents).await.map_err(|error| { + Status::internal(format!( + "write Docker boundary file {}: {error}", + path.display() + )) + })?; + openshell_core::paths::set_file_owner_only(path).map_err(|error| { + Status::internal(format!( + "restrict Docker boundary file {}: {error}", + path.display() + )) + }) +} + +async fn prepare_docker_boundary_files( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + container_id: &str, + image: &DockerImageMetadata, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + let boundary_directory = docker_boundary_mount_dir(sandbox, config); + let bootstrap_token = random_boundary_token(); + let host_gateway_ip = Some(match config.gateway_route { + DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), + DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), + }); + let provisioning = isolation::DockerBoundarySpec { + boundary_id: sandbox.id.clone(), + bootstrap_token, + container_id: container_id.to_string(), + image_identity: image.id.clone(), + listener_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH), + control_socket: boundary_directory.join("control.sock"), + host_gateway_ip, + oci_user: image.user.clone(), + trusted_runtime_root: PathBuf::from(SUPERVISOR_RUNTIME_MOUNT_PATH), + child_env: docker_child_environment(sandbox), + } + .provision(); + write_docker_boundary_file( + &boundary_directory.join("config.json"), + &provisioning + .boundary_config + .encode() + .map_err(|error| Status::internal(error.to_string()))?, + ) + .await?; + let descriptor = provisioning + .topology + .descriptor(DRIVER_ADMITTED_BACKEND) + .map_err(|error| Status::internal(error.to_string()))?; + write_docker_boundary_file(&directory.join(TOPOLOGY_PAYLOAD_FILE), &descriptor.payload).await?; + let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( + sandbox.spec.as_ref(), + ) + .map_err(|error| Status::internal(format!("encode Docker main process spec: {error}")))?; + write_docker_boundary_file( + &directory.join(MAIN_PROCESS_SPEC_FILE), + main_process_spec.as_bytes(), + ) + .await?; + let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) + .map_err(Status::failed_precondition)?; + write_docker_boundary_file( + &directory.join(WORKSPACE_ROOT_FILE), + workspace_root.as_bytes(), + ) + .await?; + Ok(provisioning.topology) } -fn cleanup_sandbox_token_file_for_delete( +async fn read_docker_boundary_topology( sandbox_id: &str, - pending: Option<&PendingSandboxRecord>, config: &DockerDriverRuntimeConfig, -) { - if !sandbox_id.is_empty() { - cleanup_sandbox_token_file_by_id(sandbox_id, config); - } else if let Some(record) = pending { - cleanup_sandbox_token_file(&record.sandbox, config); +) -> Result, Status> { + let path = docker_boundary_state_dir_by_id(sandbox_id, config)?.join(TOPOLOGY_PAYLOAD_FILE); + let bytes = match tokio::fs::read(&path).await { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(Status::internal(format!( + "read Docker boundary topology {}: {error}", + path.display() + ))); + } + }; + serde_json::from_slice(&bytes).map(Some).map_err(|error| { + Status::internal(format!( + "decode Docker boundary topology {}: {error}", + path.display() + )) + }) +} + +async fn spawn_docker_control_process( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + topology: &BoundaryTopology, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + let descriptor = topology + .descriptor(DRIVER_ADMITTED_BACKEND) + .map_err(|error| Status::internal(error.to_string()))?; + let main_process_spec = tokio::fs::read_to_string(directory.join(MAIN_PROCESS_SPEC_FILE)) + .await + .map_err(|error| Status::internal(format!("read Docker main process spec: {error}")))?; + let workspace_root = tokio::fs::read_to_string(directory.join(WORKSPACE_ROOT_FILE)) + .await + .map_err(|error| Status::internal(format!("read Docker workspace root: {error}")))?; + let stdout = std::fs::File::create(directory.join("supervisor.log")) + .map_err(|error| Status::internal(format!("create Docker supervisor log: {error}")))?; + let stderr = std::fs::File::create(directory.join("supervisor.err.log")).map_err(|error| { + Status::internal(format!("create Docker supervisor error log: {error}")) + })?; + let mut command = Command::new(&config.control_bin); + command + .kill_on_drop(true) + .stdin(Stdio::null()) + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)) + .arg("--mode=control") + .arg(format!( + "--topology-backend-name={}", + descriptor.backend_name + )) + .arg(format!("--topology-version={}", descriptor.version)) + .arg("--topology-payload-file") + .arg(directory.join(TOPOLOGY_PAYLOAD_FILE)) + .arg("--workdir") + .arg(workspace_root) + .env( + openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND, + DRIVER_ADMITTED_BACKEND, + ) + .env( + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, + main_process_spec, + ) + .env( + openshell_core::sandbox_env::ENDPOINT, + &config.host_grpc_endpoint, + ) + .env(openshell_core::sandbox_env::SANDBOX_ID, &sandbox.id) + .env(openshell_core::sandbox_env::SANDBOX, &sandbox.name) + .env( + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, + sandbox_token_host_path(sandbox, config)?, + ) + .env( + openshell_core::sandbox_env::SSH_SOCKET_PATH, + docker_control_ssh_socket(sandbox, config), + ) + .env( + openshell_core::sandbox_env::PROXY_TLS_DIR, + directory.join("proxy-tls"), + ) + .env(openshell_core::sandbox_env::SANDBOX_UID, "") + .env(openshell_core::sandbox_env::SANDBOX_GID, "") + .env(openshell_core::sandbox_env::OCI_IMAGE_USER, "") + .env( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY, + ) + .env( + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), + ) + .env( + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value(), + ); + if let Some(server_name) = config.gateway_tls_server_name.as_deref() { + command.env( + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + server_name, + ); + } + if let Some(tls) = &config.guest_tls { + command + .env(openshell_core::sandbox_env::TLS_CA, &tls.ca) + .env(openshell_core::sandbox_env::TLS_CERT, &tls.cert) + .env(openshell_core::sandbox_env::TLS_KEY, &tls.key); + } + let mut child = command.spawn().map_err(|error| { + Status::internal(format!( + "start Docker control supervisor {}: {error}", + config.control_bin.display() + )) + })?; + let sandbox_id = sandbox.id.clone(); + let (shutdown, mut shutdown_requested) = oneshot::channel(); + let task = tokio::spawn(async move { + tokio::select! { + result = child.wait() => match result { + Ok(status) => info!(%sandbox_id, %status, "Docker control supervisor exited"), + Err(error) => warn!(%sandbox_id, %error, "Failed to wait for Docker control supervisor"), + }, + _ = &mut shutdown_requested => { + if let Err(error) = child.start_kill() { + warn!(%sandbox_id, %error, "Failed to terminate Docker control supervisor"); + } + let _ = child.wait().await; + } + } + }); + Ok(DockerControlProcess { + shutdown: Some(shutdown), + task, + }) +} + +async fn stop_docker_control_process(mut process: DockerControlProcess) { + if let Some(shutdown) = process.shutdown.take() { + let _ = shutdown.send(()); } + let _ = process.task.await; } -fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { - let Ok(path) = sandbox_token_host_path_by_id(sandbox_id, config) else { +fn cleanup_docker_boundary_state(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { + cleanup_docker_boundary_state_by_id(&sandbox.id, config); +} + +fn cleanup_docker_boundary_state_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { + let Ok(directory) = docker_boundary_state_dir_by_id(sandbox_id, config) else { return; }; - if let Err(err) = std::fs::remove_file(&path) - && err.kind() != std::io::ErrorKind::NotFound + let boundary_directory = docker_boundary_mount_dir_by_id(sandbox_id, config); + let ssh_socket = docker_control_ssh_socket_by_id(sandbox_id, config); + if let Err(error) = std::fs::remove_file(&ssh_socket) + && error.kind() != std::io::ErrorKind::NotFound + { + warn!( + %sandbox_id, + path = %ssh_socket.display(), + %error, + "Failed to remove Docker control SSH socket" + ); + } + if let Err(error) = std::fs::remove_dir_all(&boundary_directory) + && error.kind() != std::io::ErrorKind::NotFound + { + warn!( + %sandbox_id, + path = %boundary_directory.display(), + %error, + "Failed to remove Docker boundary runtime directory" + ); + } + if let Err(error) = std::fs::remove_dir_all(&directory) + && error.kind() != std::io::ErrorKind::NotFound { warn!( - sandbox_id = %sandbox_id, - path = %path.display(), - error = %err, - "Failed to remove Docker sandbox token file" + %sandbox_id, + path = %directory.display(), + %error, + "Failed to remove Docker boundary state directory" ); } - if let Some(dir) = path.parent() { - let _ = std::fs::remove_dir(dir); +} + +fn random_boundary_token() -> String { + let mut token = String::with_capacity(64); + for byte in rand::random::<[u8; 32]>() { + write!(&mut token, "{byte:02x}").expect("writing to String cannot fail"); } + token } #[cfg(test)] @@ -2819,6 +3303,7 @@ fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig build_environment_for_oci_user(sandbox, config, "") } +#[cfg(test)] fn build_environment_for_oci_user( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, @@ -2834,16 +3319,10 @@ fn build_environment_for_oci_user( ), ]); - if let Some(spec) = sandbox.spec.as_ref() { - let mut user_env = HashMap::new(); - if let Some(template) = spec.template.as_ref() { - user_env.extend(template.environment.clone()); - } - user_env.extend(spec.environment.clone()); + let user_env = docker_child_environment(sandbox); + if !user_env.is_empty() { environment.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { + if let Ok(json) = serde_json::to_string(&user_env) { environment.insert( openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), json, @@ -2939,6 +3418,36 @@ fn build_environment_for_oci_user( .collect() } +fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap { + let mut environment = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .map_or_else(HashMap::new, |template| template.environment.clone()); + if let Some(spec) = sandbox.spec.as_ref() { + environment.extend(spec.environment.clone()); + } + environment +} + +fn build_boundary_environment( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Vec { + vec![ + format!( + "{}={}", + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level) + ), + format!( + "{}={}", + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value() + ), + ] +} + fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory { CdiGpuInventory::new( info.discovered_devices @@ -3044,6 +3553,8 @@ fn build_container_create_body_for_image( .map_err(Status::failed_precondition)?; driver_mounts::validate_workspace_control_path(&workspace_root, &config.ssh_socket_path) .map_err(Status::failed_precondition)?; + driver_mounts::validate_workspace_control_path(&workspace_root, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; for volume in &image.volumes { driver_mounts::validate_container_mount_target(volume).map_err(|error| { Status::failed_precondition(format!( @@ -3057,6 +3568,8 @@ fn build_container_create_body_for_image( })?; driver_mounts::validate_mount_control_path(volume, &config.ssh_socket_path) .map_err(Status::failed_precondition)?; + driver_mounts::validate_mount_control_path(volume, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; } for mount in &driver_config.mounts { let target = match mount { @@ -3069,6 +3582,8 @@ fn build_container_create_body_for_image( .map_err(Status::failed_precondition)?; driver_mounts::validate_mount_control_path(target, &config.ssh_socket_path) .map_err(Status::failed_precondition)?; + driver_mounts::validate_mount_control_path(target, BOUNDARY_MOUNT_PATH) + .map_err(Status::failed_precondition)?; } let user_mounts = docker_driver_mounts(driver_config)?; let user_bind_strings = docker_driver_bind_strings(driver_config)?; @@ -3105,11 +3620,14 @@ fn build_container_create_body_for_image( // The image workspace may need to be created or rejected by the // supervisor, so do not let the OCI runtime chdir there first. working_dir: Some("/".to_string()), - env: Some(build_environment_for_oci_user(sandbox, config, &image.user)), + env: Some(build_boundary_environment(sandbox, config)), entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), - // Replace the image CMD with the supervisor's resolved workspace - // argument so Docker cannot append inherited image arguments. - cmd: Some(vec!["--workdir".to_string(), workspace_root]), + // The image cannot append inherited arguments or select either role. + cmd: Some(vec![ + "--mode=boundary".to_string(), + "--boundary-config".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ]), labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, @@ -3117,7 +3635,7 @@ fn build_container_create_body_for_image( pids_limit: docker_pids_limit(config.sandbox_pids_limit)?, device_requests, binds: { - let mut binds = build_binds(sandbox, config)?; + let mut binds = build_binds(sandbox, config); binds.extend(user_bind_strings); Some(binds) }, @@ -3142,16 +3660,14 @@ fn build_container_create_body_for_image( // container layer is redundant relative to those controls // and conflicts with them in this case. security_opt: Some(vec!["apparmor=unconfined".to_string()]), - network_mode: Some(config.network_name.clone()), - extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), + // The container itself has no Docker network. The boundary role + // gives only the nested workload namespace an egress path, and + // carries each accepted stream to host control over the private + // Unix socket. + network_mode: Some("none".to_string()), ..Default::default() }), - networking_config: Some(NetworkingConfig { - endpoints_config: Some(HashMap::from([( - config.network_name.clone(), - EndpointSettings::default(), - )])), - }), + networking_config: None, ..Default::default() }) } @@ -3170,6 +3686,7 @@ fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<() Ok(()) } +#[cfg(test)] fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { let Ok(mut url) = Url::parse(endpoint) else { return endpoint.to_string(); @@ -3182,6 +3699,30 @@ fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> endpoint.to_string() } +fn docker_host_openshell_endpoint( + endpoint: &str, + route: &DockerGatewayRoute, +) -> CoreResult { + let mut url = Url::parse(endpoint) + .map_err(|error| Error::config(format!("invalid docker grpc_endpoint: {error}")))?; + if !matches!( + url.host_str(), + Some(HOST_OPENSHELL_INTERNAL | HOST_DOCKER_INTERNAL) + ) { + return Ok(url.to_string()); + } + let host = match route { + DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), + DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), + }; + url.set_host(Some(&host.to_string())).map_err(|error| { + Error::config(format!( + "failed to map Docker gateway alias to its host listener: {error}" + )) + })?; + Ok(url.to_string()) +} + fn docker_network_name(config: &DockerComputeConfig) -> String { let name = config.network_name.trim(); if name.is_empty() { @@ -3301,6 +3842,7 @@ fn uses_host_gateway_alias(info: &SystemInfo) -> bool { }) } +#[cfg(test)] fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { match route { DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ @@ -3873,6 +4415,57 @@ pub(crate) async fn resolve_supervisor_bin( } } +fn resolve_control_supervisor_bin(boundary_bin: &Path) -> CoreResult { + if let Ok(current_exe) = std::env::current_exe() + && let Some(parent) = current_exe.parent() + { + let sibling = parent.join("openshell-sandbox"); + if sibling.is_file() { + return canonicalize_existing_file(&sibling, "Docker host control supervisor"); + } + } + for candidate in [ + PathBuf::from("target/release/openshell-sandbox"), + PathBuf::from("target/debug/openshell-sandbox"), + ] { + if candidate.is_file() { + return canonicalize_existing_file(&candidate, "Docker host control supervisor"); + } + } + if cfg!(target_os = "linux") { + return Ok(boundary_bin.to_path_buf()); + } + Err(Error::config( + "Docker control mode requires a native openshell-sandbox binary beside the gateway", + )) +} + +async fn resolve_supervisor_runtime( + docker: &Docker, + docker_config: &DockerComputeConfig, + supervisor_bin: &Path, +) -> CoreResult { + if let Some(runtime) = supervisor_bin + .parent() + .map(|parent| parent.join("openshell-runtime")) + && validate_supervisor_runtime(&runtime).is_ok() + { + return Ok(runtime); + } + + let image = docker_config + .supervisor_image + .clone() + .unwrap_or_else(openshell_core::config::default_supervisor_image); + let extracted_bin = extract_supervisor_bin_from_image(docker, &image).await?; + let runtime = extracted_bin + .parent() + .expect("cache path has a parent") + .join("openshell-runtime"); + validate_supervisor_runtime(&runtime)?; + Ok(runtime) +} + fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { match daemon_arch { "arm64" => vec![PathBuf::from( @@ -3945,6 +4538,8 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core .map_err(Error::config)?; if cache_path.is_file() { validate_linux_elf_binary(&cache_path).map_err(Error::config)?; + ensure_cached_supervisor_runtime(docker, image, cache_path.parent().expect("cache parent")) + .await?; return Ok(cache_path); } @@ -3958,9 +4553,95 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; write_cache_binary_atomic(&cache_path, &binary_bytes).map_err(Error::config)?; validate_linux_elf_binary(&cache_path).map_err(Error::config)?; + ensure_cached_supervisor_runtime(docker, image, cache_path.parent().expect("cache parent")) + .await?; Ok(cache_path) } +fn validate_supervisor_runtime(runtime: &Path) -> CoreResult<()> { + if !runtime.is_dir() { + return Err(Error::config(format!( + "trusted supervisor helper runtime '{}' is missing", + runtime.display() + ))); + } + let has_ip = ["usr/sbin/ip", "sbin/ip", "usr/bin/ip", "bin/ip"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_nft = ["usr/sbin/nft", "sbin/nft", "usr/bin/nft"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_loader = std::fs::read_dir(runtime.join("lib")) + .ok() + .into_iter() + .flatten() + .filter_map(Result::ok) + .any(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("ld-musl-") && name.ends_with(".so.1")) + }); + if !has_ip || !has_nft || !has_loader { + return Err(Error::config(format!( + "trusted supervisor helper runtime '{}' is incomplete", + runtime.display() + ))); + } + Ok(()) +} + +async fn ensure_cached_supervisor_runtime( + docker: &Docker, + image: &str, + cache_dir: &Path, +) -> CoreResult<()> { + let runtime = cache_dir.join("openshell-runtime"); + if validate_supervisor_runtime(&runtime).is_ok() { + return Ok(()); + } + + let archive = extract_supervisor_runtime_archive(docker, image).await?; + let staging = tempfile::Builder::new() + .prefix(".openshell-runtime-") + .tempdir_in(cache_dir) + .map_err(|error| Error::config(format!("create runtime staging directory: {error}")))?; + let mut archive = tar::Archive::new(std::io::Cursor::new(archive)); + for entry in archive + .entries() + .map_err(|error| Error::config(format!("open supervisor runtime archive: {error}")))? + { + let mut entry = entry + .map_err(|error| Error::config(format!("read supervisor runtime archive: {error}")))?; + let kind = entry.header().entry_type(); + if !kind.is_file() && !kind.is_dir() { + return Err(Error::config( + "supervisor runtime archive contains a link or special file", + )); + } + if !entry.unpack_in(staging.path()).map_err(|error| { + Error::config(format!("extract supervisor runtime archive: {error}")) + })? { + return Err(Error::config( + "supervisor runtime archive contains a path outside its root", + )); + } + } + let extracted = staging.path().join("openshell-runtime"); + validate_supervisor_runtime(&extracted)?; + match std::fs::rename(&extracted, &runtime) { + Ok(()) => {} + Err(_) if validate_supervisor_runtime(&runtime).is_ok() => {} + Err(error) => { + return Err(Error::config(format!( + "install trusted supervisor runtime '{}': {error}", + runtime.display() + ))); + } + } + validate_supervisor_runtime(&runtime) +} + async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { let mut stream = docker.create_image( Some(CreateImageOptions { @@ -3984,6 +4665,19 @@ async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { /// binary as a tar archive, and return the untarred file bytes. The /// container is always removed, even on error paths. async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { + extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_BINARY_PATH, true).await +} + +async fn extract_supervisor_runtime_archive(docker: &Docker, image: &str) -> CoreResult> { + extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_RUNTIME_PATH, false).await +} + +async fn extract_supervisor_path_archive( + docker: &Docker, + image: &str, + path: &str, + extract_single_file: bool, +) -> CoreResult> { let container_name = temp_extract_container_name(); docker .create_container( @@ -4007,7 +4701,8 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe })?; // Always tear down the extractor container, even if extraction fails. - let result = download_binary_from_container(docker, &container_name).await; + let result = + download_path_from_container(docker, &container_name, path, extract_single_file).await; if let Err(remove_err) = docker .remove_container( &container_name, @@ -4024,12 +4719,14 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe result } -async fn download_binary_from_container( +async fn download_path_from_container( docker: &Docker, container_name: &str, + path: &str, + extract_single_file: bool, ) -> CoreResult> { let options = DownloadFromContainerOptionsBuilder::default() - .path(SUPERVISOR_IMAGE_BINARY_PATH) + .path(path) .build(); let mut stream = docker.download_from_container(container_name, Some(options)); @@ -4043,11 +4740,15 @@ async fn download_binary_from_container( tar_bytes.extend_from_slice(&chunk); } - extract_first_tar_entry(&tar_bytes).map_err(|err| { - Error::config(format!( - "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", - )) - }) + if extract_single_file { + extract_first_tar_entry(&tar_bytes).map_err(|err| { + Error::config(format!( + "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", + )) + }) + } else { + Ok(tar_bytes) + } } fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 07fb2b0ffb..3a6b5dcd80 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -98,7 +98,6 @@ fn runtime_config() -> DockerDriverRuntimeConfig { image_pull_policy: String::new(), sandbox_namespace: "default".to_string(), grpc_endpoint: "https://localhost:8443".to_string(), - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), gateway_route: DockerGatewayRoute::Bridge { bind_address: SocketAddr::new( IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), @@ -114,6 +113,10 @@ fn runtime_config() -> DockerDriverRuntimeConfig { stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: "info".to_string(), supervisor_bin: PathBuf::from("/tmp/openshell-sandbox"), + supervisor_runtime: PathBuf::from("/tmp/openshell-runtime"), + control_bin: PathBuf::from("/tmp/openshell-sandbox"), + host_grpc_endpoint: "https://localhost:8443".to_string(), + gateway_tls_server_name: None, guest_tls: Some(DockerGuestTlsPaths { ca: PathBuf::from("/tmp/ca.crt"), cert: PathBuf::from("/tmp/tls.crt"), @@ -165,9 +168,35 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr allow_all_default_gpu, )), lifecycle_event_fences: DockerLifecycleEventFences::default(), + control_processes: Arc::new(tokio::sync::Mutex::new(HashMap::new())), } } +#[test] +fn docker_boundary_socket_path_is_short_and_namespaced() { + let sandbox = test_sandbox(); + let config = runtime_config(); + let directory = docker_boundary_mount_dir(&sandbox, &config); + let socket = directory.join("control.sock"); + let ssh_socket = docker_control_ssh_socket(&sandbox, &config); + + assert!(directory.starts_with(DOCKER_BOUNDARY_RUNTIME_ROOT)); + assert!(socket.as_os_str().len() < 108); + assert!(ssh_socket.starts_with(DOCKER_BOUNDARY_RUNTIME_ROOT)); + assert!(ssh_socket.as_os_str().len() < 108); + + let mut other_namespace = runtime_config(); + other_namespace.sandbox_namespace = "other".to_string(); + assert_ne!( + directory, + docker_boundary_mount_dir(&sandbox, &other_namespace) + ); + assert_ne!( + ssh_socket, + docker_control_ssh_socket(&sandbox, &other_namespace) + ); +} + #[tokio::test] async fn tracing_in_process_service_preserves_the_driver_rpc_server_boundary() { use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; @@ -1179,12 +1208,17 @@ fn container_creation_uses_inspected_immutable_image() { assert_eq!(body.working_dir.as_deref(), Some("/")); assert_eq!( body.cmd.as_deref(), - Some(&["--workdir".to_string(), "/workspace/project".to_string()][..]) + Some( + &[ + "--mode=boundary".to_string(), + "--boundary-config".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ][..] + ) ); - assert!(body.env.unwrap().contains(&format!( - "{}=1234:1235", - openshell_core::sandbox_env::OCI_IMAGE_USER - ))); + assert!(body.env.unwrap().iter().all(|entry| { + !entry.starts_with(&format!("{}=", openshell_core::sandbox_env::OCI_IMAGE_USER)) + })); } #[test] @@ -1412,20 +1446,19 @@ fn build_environment_keeps_telemetry_toggle_driver_controlled() { } #[test] -fn build_binds_uses_docker_tls_directory() { - let binds = build_binds(&test_sandbox(), &runtime_config()).unwrap(); +fn build_binds_exposes_only_boundary_runtime_material() { + let binds = build_binds(&test_sandbox(), &runtime_config()); let targets = binds .iter() .filter_map(|bind| bind.split(':').nth(1).map(String::from)) .collect::>(); assert!(targets.contains(&SUPERVISOR_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_CA_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_CERT_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_KEY_MOUNT_PATH.to_string())); + assert!(targets.contains(&SUPERVISOR_RUNTIME_MOUNT_PATH.to_string())); + assert!(targets.contains(&BOUNDARY_MOUNT_PATH.to_string())); assert!( - targets + !targets .iter() - .all(|target| target.starts_with(TLS_MOUNT_DIR) || target == SUPERVISOR_MOUNT_PATH) + .any(|target| target.starts_with(TLS_MOUNT_DIR)) ); } @@ -2013,7 +2046,7 @@ fn managed_container_label_filters_include_gateway_namespace() { } #[test] -fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { +fn build_container_create_body_replaces_inherited_cmd_with_boundary_mode() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); assert_eq!( @@ -2022,7 +2055,11 @@ fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { ); assert_eq!( create_body.cmd, - Some(vec!["--workdir".to_string(), "/sandbox".to_string()]) + Some(vec![ + "--mode=boundary".to_string(), + "--boundary-config".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ]) ); assert_eq!( create_body @@ -2040,25 +2077,9 @@ fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { host_config.security_opt.as_ref(), Some(&vec!["apparmor=unconfined".to_string()]) ); - assert_eq!( - host_config.network_mode.as_deref(), - Some(DEFAULT_DOCKER_NETWORK_NAME) - ); - assert_eq!( - host_config.extra_hosts.as_ref(), - Some(&vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ]) - ); - assert_eq!( - create_body - .networking_config - .as_ref() - .and_then(|config| config.endpoints_config.as_ref()) - .and_then(|endpoints| endpoints.get(DEFAULT_DOCKER_NETWORK_NAME)), - Some(&EndpointSettings::default()) - ); + assert_eq!(host_config.network_mode.as_deref(), Some("none")); + assert!(host_config.extra_hosts.is_none()); + assert!(create_body.networking_config.is_none()); } #[test] @@ -2507,23 +2528,16 @@ fn require_sandbox_identifier_rejects_when_id_and_name_are_empty() { } #[test] -fn build_container_create_body_uses_bridge_network() { +fn build_container_create_body_disables_docker_networking() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = create_body.host_config.expect("host_config is populated"); assert_eq!( host_config.network_mode, - Some(DEFAULT_DOCKER_NETWORK_NAME.to_string()), - "sandbox should join the driver-managed bridge network" - ); - assert_eq!( - host_config.extra_hosts, - Some(vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ]), - "sandbox should expose stable host aliases for gateway callbacks" + Some("none".to_string()), + "only the nested workload namespace should receive mediated egress" ); + assert!(host_config.extra_hosts.is_none()); } #[test] From c23f9c314e19ad36ee564647e7f3ec444ecc2b6d Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 2 Sep 2026 22:06:19 -0700 Subject: [PATCH 2/6] chore(docker): update workspace lockfile Signed-off-by: Drew Newberry --- Cargo.lock | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 0a90c0fc30..cb9e9fb8b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3918,13 +3918,16 @@ dependencies = [ "http 1.4.0", "miette", "openshell-core", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", "opentelemetry", "opentelemetry_sdk", "prost-types", + "rand 0.9.4", "serde", "serde_json", + "sha2 0.10.9", "tar", "temp-env", "tempfile", From 447802879d6af22a383213b3aade79e2578b7664 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 01:50:38 -0700 Subject: [PATCH 3/6] fix(docker): harden boundary control lifecycle Signed-off-by: Drew Newberry --- Cargo.lock | 1 + crates/openshell-driver-docker/Cargo.toml | 1 + crates/openshell-driver-docker/src/lib.rs | 392 +++++++++++++++++--- crates/openshell-driver-docker/src/tests.rs | 100 ++++- 4 files changed, 442 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb9e9fb8b3..59a8c1424f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3925,6 +3925,7 @@ dependencies = [ "opentelemetry_sdk", "prost-types", "rand 0.9.4", + "rustix 1.1.4", "serde", "serde_json", "sha2 0.10.9", diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index b66a7f3274..71677296d3 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -41,6 +41,7 @@ tower-http = { workspace = true } http = { workspace = true } rand = { workspace = true } sha2 = { workspace = true } +rustix = { workspace = true } tar = "0.4" tempfile = "3" diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 981051f16d..d714886f49 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -63,6 +63,8 @@ use std::collections::{HashMap, HashSet}; use std::fmt::Write as _; use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +#[cfg(unix)] +use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _, PermissionsExt as _}; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::process::Stdio; @@ -88,7 +90,7 @@ const SUPERVISOR_IMAGE_RUNTIME_PATH: &str = "/openshell-runtime"; const BOUNDARY_MOUNT_PATH: &str = "/run/openshell/boundary"; const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/run/openshell/boundary/config.json"; const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/run/openshell/boundary/control.sock"; -const DOCKER_BOUNDARY_RUNTIME_ROOT: &str = "/tmp/openshell-docker-boundaries"; +const DOCKER_BOUNDARY_RUNTIME_DIR: &str = "docker-boundary-runtime"; const DRIVER_ADMITTED_BACKEND: &str = "docker"; const TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload"; const MAIN_PROCESS_SPEC_FILE: &str = "main-process.json"; @@ -231,6 +233,7 @@ struct DockerDriverRuntimeConfig { supervisor_bin: PathBuf, supervisor_runtime: PathBuf, control_bin: PathBuf, + boundary_runtime_root: PathBuf, host_grpc_endpoint: String, gateway_tls_server_name: Option, guest_tls: Option, @@ -259,6 +262,7 @@ pub struct DockerComputeDriver { gpu_selector: Arc, lifecycle_event_fences: DockerLifecycleEventFences, control_processes: Arc>>, + control_failures: Arc>>, } struct DockerControlProcess { @@ -266,6 +270,17 @@ struct DockerControlProcess { task: JoinHandle<()>, } +#[derive(Clone)] +struct DockerControlFailureContext { + docker: Arc, + events: broadcast::Sender, + failures: Arc>>, + sandbox: DriverSandbox, + sandbox_namespace: String, + container_id: String, + stop_timeout_secs: u32, +} + /// Per-sandbox container exit timestamps that fence snapshots from an earlier run. /// /// Docker's polling loop can observe the stopped container before a restart and @@ -677,6 +692,7 @@ impl DockerComputeDriver { resolve_supervisor_runtime(&docker, &docker_config, &supervisor_bin).await?; let control_bin = resolve_control_supervisor_bin(&supervisor_bin)?; let guest_tls = docker_guest_tls_paths(&docker_config)?; + let boundary_runtime_root = prepare_docker_boundary_runtime_root()?; let driver = Self { docker: Arc::new(docker), @@ -694,6 +710,7 @@ impl DockerComputeDriver { supervisor_bin, supervisor_runtime, control_bin, + boundary_runtime_root, host_grpc_endpoint, gateway_tls_server_name, guest_tls, @@ -711,6 +728,7 @@ impl DockerComputeDriver { )), lifecycle_event_fences: DockerLifecycleEventFences::default(), control_processes: Arc::new(Mutex::new(HashMap::new())), + control_failures: Arc::new(Mutex::new(HashMap::new())), }; let poll_driver = driver.clone(); @@ -912,9 +930,10 @@ impl DockerComputeDriver { let container = self .find_managed_container_summary(sandbox_id, sandbox_name) .await?; - if let Some(sandbox) = + if let Some(mut sandbox) = container.and_then(|summary| sandbox_from_container_summary(&summary)) { + self.apply_control_failure(&mut sandbox).await; return Ok(Some(sandbox)); } @@ -952,6 +971,7 @@ impl DockerComputeDriver { } } } + self.apply_control_failure(&mut sandbox).await; container_sandboxes.push(sandbox); } let mut by_id = self.pending_snapshot_map().await; @@ -1068,11 +1088,9 @@ impl DockerComputeDriver { image.ref = %template.image, )) .await?; - prepare_docker_boundary_state_dir(sandbox, &self.config) - .await - .map_err(|status| { - DockerProvisioningFailure::new("BoundaryStateCreateFailed", status.message()) - })?; + prepare_docker_boundary_state_dir(sandbox, &self.config).map_err(|status| { + DockerProvisioningFailure::new("BoundaryStateCreateFailed", status.message()) + })?; let token_file_created = write_sandbox_token_file(sandbox, &self.config) .await .map_err(|status| { @@ -1199,23 +1217,28 @@ impl DockerComputeDriver { create_status_from_docker_error("start docker sandbox container", err), )); } - let control = match spawn_docker_control_process(sandbox, &self.config, &topology).await { - Ok(control) => control, - Err(status) => { - let _ = self - .docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await; - cleanup_docker_boundary_state(sandbox, &self.config); - return Err(DockerProvisioningFailure::new( - "ControlSupervisorStartFailed", - status.message(), - )); - } - }; + self.clear_control_failure(&sandbox.id).await; + let failure_context = self.control_failure_context(sandbox.clone(), created.id.clone()); + let control = + match spawn_docker_control_process(sandbox, &self.config, &topology, failure_context) + .await + { + Ok(control) => control, + Err(status) => { + let _ = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "ControlSupervisorStartFailed", + status.message(), + )); + } + }; self.replace_control_process(&sandbox.id, control).await; self.publish_docker_progress( &sandbox.id, @@ -1248,6 +1271,36 @@ impl DockerComputeDriver { } } + async fn clear_control_failure(&self, sandbox_id: &str) { + self.control_failures.lock().await.remove(sandbox_id); + } + + fn control_failure_context( + &self, + sandbox: DriverSandbox, + container_id: String, + ) -> DockerControlFailureContext { + DockerControlFailureContext { + docker: self.docker.clone(), + events: self.events.clone(), + failures: self.control_failures.clone(), + sandbox, + sandbox_namespace: self.config.sandbox_namespace.clone(), + container_id, + stop_timeout_secs: self.config.stop_timeout_secs, + } + } + + async fn apply_control_failure(&self, sandbox: &mut DriverSandbox) { + let message = self.control_failures.lock().await.get(&sandbox.id).cloned(); + if let Some(message) = message { + set_sandbox_ready_condition( + sandbox, + error_condition("ControlSupervisorExited", &message), + ); + } + } + async fn stop_control_process(&self, sandbox_id: &str) { let process = self.control_processes.lock().await.remove(sandbox_id); if let Some(process) = process { @@ -1280,7 +1333,31 @@ impl DockerComputeDriver { // topology file and remains self-managing across an upgrade. return Ok(()); }; - let process = spawn_docker_control_process(&sandbox, &self.config, &topology).await?; + let container_id = summary_container_target(container) + .ok_or_else(|| Status::internal("managed Docker container has no id or name"))?; + self.clear_control_failure(&sandbox.id).await; + let failure_context = self.control_failure_context(sandbox.clone(), container_id); + let process = match spawn_docker_control_process( + &sandbox, + &self.config, + &topology, + failure_context.clone(), + ) + .await + { + Ok(process) => process, + Err(status) => { + handle_docker_control_failure( + failure_context, + format!( + "failed to start Docker control supervisor: {}", + status.message() + ), + ) + .await; + return Err(status); + } + }; self.replace_control_process(&sandbox.id, process).await; Ok(()) } @@ -1315,10 +1392,12 @@ impl DockerComputeDriver { .await { Ok(()) => { + self.clear_control_failure(&record.sandbox.id).await; cleanup_docker_boundary_state(&record.sandbox, &self.config); return Ok(true); } Err(err) if is_not_found_error(&err) => { + self.clear_control_failure(&record.sandbox.id).await; cleanup_docker_boundary_state(&record.sandbox, &self.config); return Ok(true); } @@ -1348,10 +1427,12 @@ impl DockerComputeDriver { .await { Ok(()) => { + self.clear_control_failure(resolved_sandbox_id).await; cleanup_docker_boundary_state_by_id(resolved_sandbox_id, &self.config); Ok(true) } Err(err) if is_not_found_error(&err) => { + self.clear_control_failure(resolved_sandbox_id).await; cleanup_docker_boundary_state_by_id(resolved_sandbox_id, &self.config); Ok(pending.is_some()) } @@ -1366,6 +1447,7 @@ impl DockerComputeDriver { else { if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { self.stop_control_process(&record.sandbox.id).await; + self.clear_control_failure(&record.sandbox.id).await; if let Some(task) = record.task { task.abort(); } @@ -1385,7 +1467,7 @@ impl DockerComputeDriver { .map_or(sandbox_id, String::as_str); self.stop_control_process(resolved_sandbox_id).await; - match self + let result = match self .docker .stop_container( &target, @@ -1401,7 +1483,11 @@ impl DockerComputeDriver { Err(err) if is_not_modified_error(&err) => Ok(()), Err(err) if is_not_found_error(&err) => Err(Status::not_found("sandbox not found")), Err(err) => Err(internal_status("stop docker sandbox container", err)), + }; + if result.is_ok() { + self.clear_control_failure(resolved_sandbox_id).await; } + result } /// Start a managed sandbox container that was previously stopped. Used @@ -1594,8 +1680,9 @@ impl DockerComputeDriver { if let Some(summary) = self .find_managed_container_summary(sandbox_id, sandbox_name) .await? - && let Some(sandbox) = sandbox_from_container_summary(&summary) + && let Some(mut sandbox) = sandbox_from_container_summary(&summary) { + self.apply_control_failure(&mut sandbox).await; self.publish_sandbox_snapshot(sandbox); } Ok(()) @@ -2438,6 +2525,21 @@ fn error_condition(reason: &str, message: &str) -> DriverCondition { } } +fn set_sandbox_ready_condition(sandbox: &mut DriverSandbox, condition: DriverCondition) { + let Some(status) = sandbox.status.as_mut() else { + return; + }; + if let Some(existing) = status + .conditions + .iter_mut() + .find(|existing| existing.r#type == "Ready") + { + *existing = condition; + } else { + status.conditions.push(condition); + } +} + fn platform_event( source: &str, event_type: &str, @@ -2913,7 +3015,132 @@ fn docker_boundary_mount_dir_by_id( hasher.update([0]); hasher.update(sandbox_id.as_bytes()); let digest = format!("{:x}", hasher.finalize()); - Path::new(DOCKER_BOUNDARY_RUNTIME_ROOT).join(&digest[..32]) + config.boundary_runtime_root.join(&digest[..32]) +} + +fn prepare_docker_boundary_runtime_root() -> CoreResult { + let (base_root, require_private_base) = if let Some(runtime_root) = + std::env::var_os("XDG_RUNTIME_DIR").filter(|value| !value.is_empty()) + { + (PathBuf::from(runtime_root), true) + } else { + ( + openshell_core::paths::xdg_state_dir().map_err(|error| { + Error::config(format!("resolve Docker boundary state directory: {error}")) + })?, + false, + ) + }; + std::fs::create_dir_all(&base_root).map_err(|error| { + Error::config(format!( + "create Docker boundary runtime base {}: {error}", + base_root.display() + )) + })?; + validate_owned_directory(&base_root, require_private_base).map_err(Error::config)?; + let openshell_root = base_root.join("openshell"); + ensure_owned_directory(&openshell_root, false).map_err(Error::config)?; + + let runtime_root = openshell_root.join(DOCKER_BOUNDARY_RUNTIME_DIR); + ensure_owned_directory(&runtime_root, true).map_err(Error::config)?; + validate_docker_socket_path_length(&runtime_root).map_err(Error::config)?; + Ok(runtime_root) +} + +fn validate_docker_socket_path_length(runtime_root: &Path) -> Result<(), String> { + let longest = runtime_root.join("f".repeat(32)).with_extension("ssh"); + #[cfg(unix)] + let length = { + use std::os::unix::ffi::OsStrExt as _; + longest.as_os_str().as_bytes().len() + }; + #[cfg(not(unix))] + let length = longest.as_os_str().to_string_lossy().len(); + if length >= 108 { + return Err(format!( + "Docker boundary runtime path '{}' is too long for a Unix socket; set XDG_RUNTIME_DIR or XDG_STATE_HOME to a shorter private path", + runtime_root.display() + )); + } + Ok(()) +} + +fn ensure_owned_directory(path: &Path, require_private: bool) -> Result<(), String> { + match std::fs::create_dir(path) { + Ok(()) => { + #[cfg(unix)] + std::fs::set_permissions( + path, + std::fs::Permissions::from_mode(if require_private { 0o700 } else { 0o755 }), + ) + .map_err(|error| format!("restrict directory {}: {error}", path.display()))?; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(format!("create directory {}: {error}", path.display())), + } + validate_owned_directory(path, require_private) +} + +fn validate_owned_directory(path: &Path, require_private: bool) -> Result<(), String> { + let metadata = std::fs::symlink_metadata(path) + .map_err(|error| format!("inspect directory {}: {error}", path.display()))?; + if metadata.file_type().is_symlink() || !metadata.file_type().is_dir() { + return Err(format!( + "Docker boundary path '{}' must be a directory and must not be a symlink", + path.display() + )); + } + #[cfg(unix)] + { + let expected_uid = rustix::process::geteuid().as_raw(); + if metadata.uid() != expected_uid { + return Err(format!( + "Docker boundary path '{}' is owned by uid {}, expected {}", + path.display(), + metadata.uid(), + expected_uid + )); + } + let mode = metadata.mode() & 0o777; + let invalid_mode = if require_private { + mode != 0o700 + } else { + mode & 0o022 != 0 + }; + if invalid_mode { + return Err(format!( + "Docker boundary path '{}' has unsafe mode {mode:04o}", + path.display() + )); + } + } + Ok(()) +} + +fn remove_owned_stale_boundary_socket(path: &Path) -> Result<(), String> { + match std::fs::symlink_metadata(path) { + #[cfg(unix)] + Ok(metadata) + if metadata.file_type().is_socket() + && !metadata.file_type().is_symlink() + && metadata.uid() == rustix::process::geteuid().as_raw() + && metadata.mode() & 0o777 == 0o666 => + { + std::fs::remove_file(path) + .map_err(|error| format!("remove stale socket {}: {error}", path.display())) + } + #[cfg(not(unix))] + Ok(metadata) if !metadata.file_type().is_symlink() && metadata.file_type().is_file() => { + std::fs::remove_file(path) + .map_err(|error| format!("remove stale socket {}: {error}", path.display())) + } + Ok(_) => Err(format!( + "Docker boundary socket '{}' exists but is not an owned mode-0666 Unix socket", + path.display() + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(format!("inspect socket {}: {error}", path.display())), + } } fn docker_control_ssh_socket( @@ -2989,7 +3216,7 @@ async fn write_sandbox_token_file( Ok(true) } -async fn prepare_docker_boundary_state_dir( +fn prepare_docker_boundary_state_dir( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, ) -> Result { @@ -3001,30 +3228,17 @@ async fn prepare_docker_boundary_state_dir( )) })?; let boundary_directory = docker_boundary_mount_dir(sandbox, config); - let boundary_root = boundary_directory - .parent() - .ok_or_else(|| Status::internal("Docker boundary runtime path has no parent"))?; - openshell_core::paths::create_dir_restricted(boundary_root).map_err(|error| { - Status::internal(format!( - "create Docker boundary runtime root {}: {error}", - boundary_root.display() - )) + validate_owned_directory(&config.boundary_runtime_root, true).map_err(|error| { + Status::failed_precondition(format!("validate Docker boundary runtime root: {error}")) })?; - openshell_core::paths::create_dir_restricted(&boundary_directory).map_err(|error| { + ensure_owned_directory(&boundary_directory, true).map_err(|error| { Status::internal(format!( "create Docker boundary mount directory {}: {error}", boundary_directory.display() )) })?; let socket = boundary_directory.join("control.sock"); - if let Err(error) = tokio::fs::remove_file(&socket).await - && error.kind() != std::io::ErrorKind::NotFound - { - return Err(Status::internal(format!( - "remove stale Docker boundary socket {}: {error}", - socket.display() - ))); - } + remove_owned_stale_boundary_socket(&socket).map_err(Status::failed_precondition)?; Ok(directory) } @@ -3128,6 +3342,7 @@ async fn spawn_docker_control_process( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, topology: &BoundaryTopology, + failure_context: DockerControlFailureContext, ) -> Result { let directory = docker_boundary_state_dir(sandbox, config)?; let descriptor = topology @@ -3223,16 +3438,29 @@ async fn spawn_docker_control_process( let (shutdown, mut shutdown_requested) = oneshot::channel(); let task = tokio::spawn(async move { tokio::select! { - result = child.wait() => match result { - Ok(status) => info!(%sandbox_id, %status, "Docker control supervisor exited"), - Err(error) => warn!(%sandbox_id, %error, "Failed to wait for Docker control supervisor"), - }, + biased; _ = &mut shutdown_requested => { if let Err(error) = child.start_kill() { warn!(%sandbox_id, %error, "Failed to terminate Docker control supervisor"); } let _ = child.wait().await; } + result = child.wait() => match result { + Ok(status) => { + warn!(%sandbox_id, %status, "Docker control supervisor exited unexpectedly"); + handle_docker_control_failure( + failure_context, + format!("Docker control supervisor exited with status {status}"), + ).await; + } + Err(error) => { + warn!(%sandbox_id, %error, "Failed to wait for Docker control supervisor"); + handle_docker_control_failure( + failure_context, + format!("failed to wait for Docker control supervisor: {error}"), + ).await; + } + }, } }); Ok(DockerControlProcess { @@ -3241,6 +3469,70 @@ async fn spawn_docker_control_process( }) } +async fn handle_docker_control_failure(context: DockerControlFailureContext, message: String) { + context + .failures + .lock() + .await + .insert(context.sandbox.id.clone(), message.clone()); + + let mut snapshot = pending_sandbox_snapshot( + &context.sandbox, + &context.sandbox_namespace, + error_condition("ControlSupervisorExited", &message), + false, + ); + if let Some(status) = snapshot.status.as_mut() { + status.instance_id.clone_from(&context.container_id); + } + let _ = context.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(snapshot), + }, + )), + }); + let _ = context.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::PlatformEvent( + WatchSandboxesPlatformEvent { + sandbox_id: context.sandbox.id.clone(), + event: Some(platform_event( + "docker", + "Warning", + "ControlSupervisorExited", + format!("{message}; stopping the isolated workload container"), + )), + }, + )), + }); + + match context + .docker + .stop_container( + &context.container_id, + Some( + StopContainerOptionsBuilder::default() + .t(docker_stop_timeout_secs(context.stop_timeout_secs)) + .build(), + ), + ) + .await + { + Ok(()) => info!( + sandbox_id = %context.sandbox.id, + container_id = %context.container_id, + "Stopped Docker sandbox after control supervisor failure" + ), + Err(error) if is_not_found_error(&error) || is_not_modified_error(&error) => {} + Err(error) => warn!( + sandbox_id = %context.sandbox.id, + container_id = %context.container_id, + %error, + "Failed to stop Docker sandbox after control supervisor failure" + ), + } +} + async fn stop_docker_control_process(mut process: DockerControlProcess) { if let Some(shutdown) = process.shutdown.take() { let _ = shutdown.send(()); diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 3a6b5dcd80..955c1542dc 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -115,6 +115,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig { supervisor_bin: PathBuf::from("/tmp/openshell-sandbox"), supervisor_runtime: PathBuf::from("/tmp/openshell-runtime"), control_bin: PathBuf::from("/tmp/openshell-sandbox"), + boundary_runtime_root: PathBuf::from("/tmp/openshell-test-docker-boundaries"), host_grpc_endpoint: "https://localhost:8443".to_string(), gateway_tls_server_name: None, guest_tls: Some(DockerGuestTlsPaths { @@ -169,6 +170,7 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr )), lifecycle_event_fences: DockerLifecycleEventFences::default(), control_processes: Arc::new(tokio::sync::Mutex::new(HashMap::new())), + control_failures: Arc::new(tokio::sync::Mutex::new(HashMap::new())), } } @@ -180,9 +182,9 @@ fn docker_boundary_socket_path_is_short_and_namespaced() { let socket = directory.join("control.sock"); let ssh_socket = docker_control_ssh_socket(&sandbox, &config); - assert!(directory.starts_with(DOCKER_BOUNDARY_RUNTIME_ROOT)); + assert!(directory.starts_with(&config.boundary_runtime_root)); assert!(socket.as_os_str().len() < 108); - assert!(ssh_socket.starts_with(DOCKER_BOUNDARY_RUNTIME_ROOT)); + assert!(ssh_socket.starts_with(&config.boundary_runtime_root)); assert!(ssh_socket.as_os_str().len() < 108); let mut other_namespace = runtime_config(); @@ -197,6 +199,100 @@ fn docker_boundary_socket_path_is_short_and_namespaced() { ); } +#[cfg(unix)] +#[test] +fn docker_boundary_runtime_creates_private_directory() { + use std::os::unix::fs::MetadataExt as _; + + let temporary = TempDir::new().expect("create temporary directory"); + let boundary = temporary.path().join("boundary"); + + ensure_owned_directory(&boundary, true).expect("create private boundary directory"); + + let metadata = fs::symlink_metadata(&boundary).expect("inspect boundary directory"); + assert_eq!(metadata.mode() & 0o777, 0o700); + assert_eq!(metadata.uid(), rustix::process::geteuid().as_raw()); +} + +#[cfg(unix)] +#[test] +fn docker_boundary_runtime_rejects_preexisting_symlink() { + use std::os::unix::fs::symlink; + + let temporary = TempDir::new().expect("create temporary directory"); + let target = temporary.path().join("target"); + fs::create_dir(&target).expect("create target"); + let boundary = temporary.path().join("boundary"); + symlink(&target, &boundary).expect("create boundary symlink"); + + let error = ensure_owned_directory(&boundary, true).expect_err("symlink must be rejected"); + assert!(error.contains("must not be a symlink"), "{error}"); +} + +#[cfg(unix)] +#[test] +fn docker_boundary_runtime_rejects_permissive_existing_directory() { + use std::os::unix::fs::PermissionsExt as _; + + let temporary = TempDir::new().expect("create temporary directory"); + let boundary = temporary.path().join("boundary"); + fs::create_dir(&boundary).expect("create boundary directory"); + fs::set_permissions(&boundary, fs::Permissions::from_mode(0o755)).expect("set permissive mode"); + + let error = + ensure_owned_directory(&boundary, true).expect_err("permissive directory must be rejected"); + assert!(error.contains("unsafe mode 0755"), "{error}"); +} + +#[cfg(unix)] +#[test] +fn docker_boundary_runtime_rejects_non_socket_stale_entry() { + let temporary = TempDir::new().expect("create temporary directory"); + let socket = temporary.path().join("control.sock"); + fs::write(&socket, b"not a socket").expect("write stale entry"); + + let error = remove_owned_stale_boundary_socket(&socket) + .expect_err("non-socket stale entry must be rejected"); + assert!( + error.contains("not an owned mode-0666 Unix socket"), + "{error}" + ); +} + +#[tokio::test] +async fn control_failure_overrides_running_container_readiness() { + let driver = test_driver_with_config(runtime_config()); + driver.control_failures.lock().await.insert( + "sbx-123".to_string(), + "control exited unexpectedly".to_string(), + ); + let mut sandbox = pending_sandbox_snapshot( + &test_sandbox(), + "default", + DriverCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: "BackendReady".to_string(), + message: "Container is running".to_string(), + last_transition_time: String::new(), + }, + false, + ); + + driver.apply_control_failure(&mut sandbox).await; + + let ready = sandbox + .status + .unwrap() + .conditions + .into_iter() + .find(|condition| condition.r#type == "Ready") + .expect("ready condition"); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, "ControlSupervisorExited"); + assert!(ready.message.contains("control exited unexpectedly")); +} + #[tokio::test] async fn tracing_in_process_service_preserves_the_driver_rpc_server_boundary() { use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; From 00233be3ca1874629bcc05bc5e48f581470f2e76 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 02:14:23 -0700 Subject: [PATCH 4/6] fix(docker): close boundary lifecycle gaps Signed-off-by: Drew Newberry --- Cargo.lock | 1 + crates/openshell-driver-docker/Cargo.toml | 1 + crates/openshell-driver-docker/README.md | 29 +- crates/openshell-driver-docker/src/lib.rs | 368 +++++++++---------- crates/openshell-driver-docker/src/tests.rs | 369 ++++++++------------ 5 files changed, 343 insertions(+), 425 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59a8c1424f..4c38a39778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3916,6 +3916,7 @@ dependencies = [ "clap", "futures", "http 1.4.0", + "libc", "miette", "openshell-core", "openshell-isolation-interface", diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 71677296d3..38051327aa 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -42,6 +42,7 @@ http = { workspace = true } rand = { workspace = true } sha2 = { workspace = true } rustix = { workspace = true } +libc = "0.2" tar = "0.4" tempfile = "3" diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index bb96fd5b88..32eb195b4d 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -39,18 +39,22 @@ boundary owns the process tree, exec/signal/wait/PTY operations, loopback port forwarding, procfs binary identity, and the workload network namespace. The boundary receives no gateway JWT, client TLS private key, or policy authority. -The driver bind-mounts the boundary socket from a short, digest-keyed runtime -directory under `/tmp/openshell-docker-boundaries`. This keeps the host endpoint -below Linux's Unix-socket path limit even when the gateway state directory is -deeply nested. The full protected topology and gateway material remain in the -normal driver state directory, and sandbox cleanup removes both locations. +The driver bind-mounts the boundary socket from a short, digest-keyed directory +under `$XDG_RUNTIME_DIR/openshell/docker-boundary-runtime`, falling back to the +XDG state directory when no runtime directory exists. The driver rejects +symlinked, incorrectly owned, or overly permissive pre-existing components and +checks the final path against the Unix-socket length limit. The full protected +topology and gateway material remain in the normal driver state directory, and +sandbox cleanup removes both locations. ## Stop and Start Stop terminates host control and stops the managed container without removing it. Docker retains the container writable layer, attached volumes, labels, and -the protected driver topology. Start starts that same container and recreates -host control, so files in the resolved OCI workspace remain available. A +the protected driver topology. Stop removes the transient boundary and SSH +socket names; Start validates and removes any safe stale sockets before starting +that same container and recreating host control. Files in the resolved OCI +workspace remain available. A durably stopped sandbox is excluded from gateway startup recovery and stays stopped across gateway restarts. Delete force-removes the container and the driver-owned boundary directory. @@ -196,7 +200,11 @@ When no endpoint is configured, the driver uses `host.openshell.internal:` with the appropriate HTTP or HTTPS scheme. Set `host_gateway_ip` only when the host has an explicit, locally assigned address that containers should use for callbacks; package-managed -macOS gateways should leave it unset. +macOS gateways should leave it unset. The current boundary topology requires +the Docker daemon and gateway to share bind-mounted Unix socket inodes. The +driver fails during initialization for Docker Desktop, Colima, Lima, Rancher +Desktop, OrbStack, and other detected VM-backed daemons instead of starting a +workload whose host control channel cannot become operational. For HTTPS endpoints, the server certificate must include the endpoint host as a subject alternative name. The client TLS bundle is exposed only to host control @@ -212,7 +220,10 @@ HTTP endpoints reject TLS material because the supervisor would not use it. The driver writes template and sandbox environment into the protected boundary config. Boundary mode exposes that map only to workload children. Host control -owns these security-critical keys: +starts with an empty inherited environment and receives only driver-owned +values. It also arms Linux `PDEATHSIG` with a parent-PID race check so a killed +gateway cannot leave an orphan host-control process. Host control owns these +security-critical keys: - `OPENSHELL_ENDPOINT` - `OPENSHELL_SANDBOX_ID` diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d714886f49..2fe5162f07 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -92,19 +92,11 @@ const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/run/openshell/boundary/config.json"; const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/run/openshell/boundary/control.sock"; const DOCKER_BOUNDARY_RUNTIME_DIR: &str = "docker-boundary-runtime"; const DRIVER_ADMITTED_BACKEND: &str = "docker"; +const LABEL_ISOLATION_TOPOLOGY: &str = "openshell.ai/isolation-topology"; +const LABEL_ISOLATION_TOPOLOGY_BOUNDARY_V1: &str = "boundary-v1"; const TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload"; const MAIN_PROCESS_SPEC_FILE: &str = "main-process.json"; const WORKSPACE_ROOT_FILE: &str = "workspace-root"; -#[cfg(test)] -const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; -#[cfg(test)] -const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; -#[cfg(test)] -const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; -#[cfg(test)] -const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; -#[cfg(test)] -const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; const DOCKER_NETWORK_DRIVER: &str = "bridge"; @@ -223,8 +215,6 @@ struct DockerDriverRuntimeConfig { default_image: String, image_pull_policy: String, sandbox_namespace: String, - #[cfg(test)] - grpc_endpoint: String, gateway_route: DockerGatewayRoute, gateway_callback_bind_address: Option, ssh_socket_path: String, @@ -246,10 +236,7 @@ struct DockerDriverRuntimeConfig { #[derive(Debug, Clone, PartialEq, Eq)] enum DockerGatewayRoute { - Bridge { - bind_address: SocketAddr, - host_alias_ip: IpAddr, - }, + Bridge { bind_address: SocketAddr }, HostGateway, } @@ -629,6 +616,7 @@ impl DockerComputeDriver { let info = docker.info().await.map_err(|err| { Error::execution(format!("failed to query Docker daemon info: {err}")) })?; + validate_shared_unix_boundary_transport(&info, cfg!(target_os = "macos"))?; let supports_gpu = info .cdi_spec_dirs .as_ref() @@ -680,12 +668,6 @@ impl DockerComputeDriver { .unwrap_or_default() .to_string() }); - #[cfg(test)] - let grpc_endpoint = docker_container_openshell_endpoint( - &docker_config.grpc_endpoint, - HOST_OPENSHELL_INTERNAL, - gateway_port, - ); let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; let supervisor_runtime = @@ -700,8 +682,6 @@ impl DockerComputeDriver { default_image: docker_config.default_image.clone(), image_pull_policy: docker_config.image_pull_policy.clone(), sandbox_namespace: docker_config.sandbox_namespace.clone(), - #[cfg(test)] - grpc_endpoint, gateway_route, gateway_callback_bind_address, ssh_socket_path: docker_config.ssh_socket_path.clone(), @@ -1329,8 +1309,19 @@ impl DockerComputeDriver { stop_docker_control_process(stale).await; } let Some(topology) = read_docker_boundary_topology(&sandbox.id, &self.config).await? else { + if container_requires_boundary_control(container) { + let container_id = summary_container_target(container).ok_or_else(|| { + Status::internal("managed Docker container has no id or name") + })?; + let failure_context = self.control_failure_context(sandbox.clone(), container_id); + let status = Status::failed_precondition( + "Docker boundary topology is missing; refusing to leave the workload running without host control", + ); + handle_docker_control_failure(failure_context, status.message().to_string()).await; + return Err(status); + } // A container created by the legacy combined supervisor has no - // topology file and remains self-managing across an upgrade. + // boundary marker or topology file and remains self-managing. return Ok(()); }; let container_id = summary_container_target(container) @@ -1486,6 +1477,7 @@ impl DockerComputeDriver { }; if result.is_ok() { self.clear_control_failure(resolved_sandbox_id).await; + cleanup_docker_boundary_sockets_by_id(resolved_sandbox_id, &self.config); } result } @@ -1564,6 +1556,23 @@ impl DockerComputeDriver { self.lifecycle_event_fences .record_previous_exit(sandbox_id, previous_finished_at.as_deref()); + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + if container_requires_boundary_control(&container) { + if read_docker_boundary_topology(resolved_sandbox_id, &self.config) + .await? + .is_none() + { + return Err(Status::failed_precondition( + "Docker boundary topology is missing; refusing to start the workload without host control", + )); + } + prepare_docker_boundary_restart_sockets(resolved_sandbox_id, &self.config)?; + } + match self.docker.start_container(&target, None).await { Ok(()) => {} // Already running — race with another start path or the @@ -3117,14 +3126,19 @@ fn validate_owned_directory(path: &Path, require_private: bool) -> Result<(), St Ok(()) } -fn remove_owned_stale_boundary_socket(path: &Path) -> Result<(), String> { +fn remove_owned_stale_socket( + path: &Path, + expected_mode: u32, + allow_root_owner: bool, +) -> Result<(), String> { match std::fs::symlink_metadata(path) { #[cfg(unix)] Ok(metadata) if metadata.file_type().is_socket() && !metadata.file_type().is_symlink() - && metadata.uid() == rustix::process::geteuid().as_raw() - && metadata.mode() & 0o777 == 0o666 => + && (metadata.uid() == rustix::process::geteuid().as_raw() + || (allow_root_owner && metadata.uid() == 0)) + && metadata.mode() & 0o777 == expected_mode => { std::fs::remove_file(path) .map_err(|error| format!("remove stale socket {}: {error}", path.display())) @@ -3135,8 +3149,8 @@ fn remove_owned_stale_boundary_socket(path: &Path) -> Result<(), String> { .map_err(|error| format!("remove stale socket {}: {error}", path.display())) } Ok(_) => Err(format!( - "Docker boundary socket '{}' exists but is not an owned mode-0666 Unix socket", - path.display() + "Docker socket '{}' exists but is not an expected owned mode-{expected_mode:04o} Unix socket", + path.display(), )), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), Err(error) => Err(format!("inspect socket {}: {error}", path.display())), @@ -3238,7 +3252,9 @@ fn prepare_docker_boundary_state_dir( )) })?; let socket = boundary_directory.join("control.sock"); - remove_owned_stale_boundary_socket(&socket).map_err(Status::failed_precondition)?; + remove_owned_stale_socket(&socket, 0o666, true).map_err(Status::failed_precondition)?; + remove_owned_stale_socket(&docker_control_ssh_socket(sandbox, config), 0o600, false) + .map_err(Status::failed_precondition)?; Ok(directory) } @@ -3359,7 +3375,7 @@ async fn spawn_docker_control_process( let stderr = std::fs::File::create(directory.join("supervisor.err.log")).map_err(|error| { Status::internal(format!("create Docker supervisor error log: {error}")) })?; - let mut command = Command::new(&config.control_bin); + let mut command = new_control_command(&config.control_bin); command .kill_on_drop(true) .stdin(Stdio::null()) @@ -3469,6 +3485,35 @@ async fn spawn_docker_control_process( }) } +fn new_control_command(binary: &Path) -> Command { + let mut command = Command::new(binary); + command.env_clear(); + configure_control_parent_death(&mut command); + command +} + +#[cfg(target_os = "linux")] +fn configure_control_parent_death(command: &mut Command) { + let expected_parent = std::process::id(); + #[allow(unsafe_code)] + // SAFETY: the closure only invokes async-signal-safe syscalls and creates + // OS errors from integer codes between fork and exec. + unsafe { + command.pre_exec(move || { + if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 { + return Err(std::io::Error::last_os_error()); + } + if libc::getppid().cast_unsigned() != expected_parent { + return Err(std::io::Error::from_raw_os_error(libc::ECHILD)); + } + Ok(()) + }); + } +} + +#[cfg(not(target_os = "linux"))] +fn configure_control_parent_death(_command: &mut Command) {} + async fn handle_docker_control_failure(context: DockerControlFailureContext, message: String) { context .failures @@ -3540,6 +3585,37 @@ async fn stop_docker_control_process(mut process: DockerControlProcess) { let _ = process.task.await; } +fn prepare_docker_boundary_restart_sockets( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result<(), Status> { + let boundary_directory = docker_boundary_mount_dir_by_id(sandbox_id, config); + validate_owned_directory(&config.boundary_runtime_root, true).map_err(|error| { + Status::failed_precondition(format!("validate Docker boundary runtime root: {error}")) + })?; + validate_owned_directory(&boundary_directory, true).map_err(|error| { + Status::failed_precondition(format!("validate Docker boundary directory: {error}")) + })?; + remove_owned_stale_socket(&boundary_directory.join("control.sock"), 0o666, true) + .map_err(Status::failed_precondition)?; + remove_owned_stale_socket( + &docker_control_ssh_socket_by_id(sandbox_id, config), + 0o600, + false, + ) + .map_err(Status::failed_precondition) +} + +fn cleanup_docker_boundary_sockets_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { + if let Err(error) = prepare_docker_boundary_restart_sockets(sandbox_id, config) { + warn!( + %sandbox_id, + %error, + "Failed to clean up stopped Docker boundary sockets" + ); + } +} + fn cleanup_docker_boundary_state(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { cleanup_docker_boundary_state_by_id(&sandbox.id, config); } @@ -3549,17 +3625,7 @@ fn cleanup_docker_boundary_state_by_id(sandbox_id: &str, config: &DockerDriverRu return; }; let boundary_directory = docker_boundary_mount_dir_by_id(sandbox_id, config); - let ssh_socket = docker_control_ssh_socket_by_id(sandbox_id, config); - if let Err(error) = std::fs::remove_file(&ssh_socket) - && error.kind() != std::io::ErrorKind::NotFound - { - warn!( - %sandbox_id, - path = %ssh_socket.display(), - %error, - "Failed to remove Docker control SSH socket" - ); - } + cleanup_docker_boundary_sockets_by_id(sandbox_id, config); if let Err(error) = std::fs::remove_dir_all(&boundary_directory) && error.kind() != std::io::ErrorKind::NotFound { @@ -3590,126 +3656,6 @@ fn random_boundary_token() -> String { token } -#[cfg(test)] -fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { - build_environment_for_oci_user(sandbox, config, "") -} - -#[cfg(test)] -fn build_environment_for_oci_user( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - oci_user: &str, -) -> Vec { - let mut environment = HashMap::from([ - ("HOME".to_string(), "/root".to_string()), - ("PATH".to_string(), SUPERVISOR_PATH.to_string()), - ("TERM".to_string(), "xterm".to_string()), - ( - "OPENSHELL_LOG_LEVEL".to_string(), - openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), - ), - ]); - - let user_env = docker_child_environment(sandbox); - if !user_env.is_empty() { - environment.extend(user_env.clone()); - if let Ok(json) = serde_json::to_string(&user_env) { - environment.insert( - openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), - json, - ); - } - } - - environment.insert( - openshell_core::sandbox_env::ENDPOINT.to_string(), - config.grpc_endpoint.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_ID.to_string(), - sandbox.id.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX.to_string(), - sandbox.name.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), - config.ssh_socket_path.clone(), - ); - let main_process = - openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(sandbox.spec.as_ref()) - .expect("main process config serialization cannot fail"); - environment.insert( - openshell_core::sandbox_env::MAIN_PROCESS_SPEC.to_string(), - main_process, - ); - environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - openshell_core::telemetry::enabled_env_value().to_string(), - ); - environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), - ); - // The root supervisor executes namespace helpers during bootstrap; keep - // their search path driver-owned even when the template/spec set PATH. - environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); - if config.guest_tls.is_some() { - environment.insert( - openshell_core::sandbox_env::TLS_CA.to_string(), - TLS_CA_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_CERT.to_string(), - TLS_CERT_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_KEY.to_string(), - TLS_KEY_MOUNT_PATH.to_string(), - ); - } - - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - // Prevent user-supplied environment from overriding the TLS server name - // the supervisor verifies — a sandbox user who can redirect the gateway - // hostname could otherwise present a certificate for a name they control - // and intercept the sandbox JWT. - environment.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - environment.insert( - openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), - oci_user.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_UID.to_string(), - String::new(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_GID.to_string(), - String::new(), - ); - - // Gateway-minted sandbox JWT. Keep the raw bearer out of container - // metadata; the supervisor reads it from this driver-owned bind mount. - if let Some(spec) = sandbox.spec.as_ref() - && !spec.sandbox_token.is_empty() - { - environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), - SANDBOX_TOKEN_MOUNT_PATH.to_string(), - ); - } - - let mut pairs = environment.into_iter().collect::>(); - pairs.sort_by(|left, right| left.0.cmp(&right.0)); - pairs - .into_iter() - .map(|(key, value)| format!("{key}={value}")) - .collect() -} - fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap { let mut environment = sandbox .spec @@ -3719,6 +3665,26 @@ fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap if let Some(spec) = sandbox.spec.as_ref() { environment.extend(spec.environment.clone()); } + for protected in [ + openshell_core::sandbox_env::ENDPOINT, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX, + openshell_core::sandbox_env::SANDBOX_GID, + openshell_core::sandbox_env::SANDBOX_ID, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, + openshell_core::sandbox_env::SANDBOX_UID, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + openshell_core::sandbox_env::TLS_CA, + openshell_core::sandbox_env::TLS_CERT, + openshell_core::sandbox_env::TLS_KEY, + openshell_core::sandbox_env::USER_ENVIRONMENT, + ] { + environment.remove(protected); + } environment } @@ -3905,6 +3871,10 @@ fn build_container_create_body_for_image( LABEL_SANDBOX_NAMESPACE.to_string(), config.sandbox_namespace.clone(), ); + labels.insert( + LABEL_ISOLATION_TOPOLOGY.to_string(), + LABEL_ISOLATION_TOPOLOGY_BOUNDARY_V1.to_string(), + ); Ok(ContainerCreateBody { image: Some(image.id.clone()), @@ -3978,17 +3948,17 @@ fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<() Ok(()) } -#[cfg(test)] -fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); - }; - - if url.set_host(Some(host)).is_ok() && url.set_port(Some(port)).is_ok() { - return url.to_string(); - } - - endpoint.to_string() +fn container_requires_boundary_control(container: &ContainerSummary) -> bool { + container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_TOPOLOGY)) + .is_some_and(|value| value == LABEL_ISOLATION_TOPOLOGY_BOUNDARY_V1) + || container.command.as_deref().is_some_and(|command| { + command + .split_ascii_whitespace() + .any(|argument| argument == "--mode=boundary") + }) } fn docker_host_openshell_endpoint( @@ -4060,7 +4030,6 @@ fn docker_gateway_route_for_host( if let Some(host_alias_ip) = host_gateway_ip { return DockerGatewayRoute::Bridge { bind_address: SocketAddr::new(host_alias_ip, port), - host_alias_ip, }; } @@ -4069,7 +4038,6 @@ fn docker_gateway_route_for_host( } else { DockerGatewayRoute::Bridge { bind_address: SocketAddr::new(bridge_gateway_ip, port), - host_alias_ip: bridge_gateway_ip, } } } @@ -4094,6 +4062,18 @@ fn host_runtime_requires_host_gateway_alias() -> bool { cfg!(target_os = "macos") } +fn validate_shared_unix_boundary_transport( + info: &SystemInfo, + host_requires_host_gateway_alias: bool, +) -> CoreResult<()> { + if host_requires_host_gateway_alias || uses_host_gateway_alias(info) { + return Err(Error::config( + "Docker boundary control requires a local daemon that shares host Unix sockets; Docker Desktop, Colima, Lima, Rancher Desktop, OrbStack, and other VM-backed Docker daemons are not supported by this topology", + )); + } + Ok(()) +} + /// Detect Docker Desktop and behaviourally compatible runtimes - Colima, /// Lima, Rancher Desktop, and `OrbStack` - that share Docker Desktop's routing /// constraint: the bridge gateway IP is reachable from inside containers but @@ -4134,20 +4114,6 @@ fn uses_host_gateway_alias(info: &SystemInfo) -> bool { }) } -#[cfg(test)] -fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { - match route { - DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ - format!("{HOST_DOCKER_INTERNAL}:{host_alias_ip}"), - format!("{HOST_OPENSHELL_INTERNAL}:{host_alias_ip}"), - ], - DockerGatewayRoute::HostGateway => vec![ - format!("{HOST_DOCKER_INTERNAL}:host-gateway"), - format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), - ], - } -} - async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { match docker.inspect_network(network_name, None).await { Ok(network) => return validate_bridge_network(network_name, &network), @@ -4717,8 +4683,8 @@ fn resolve_control_supervisor_bin(boundary_bin: &Path) -> CoreResult { } } for candidate in [ - PathBuf::from("target/release/openshell-sandbox"), - PathBuf::from("target/debug/openshell-sandbox"), + docker_workspace_root().join("target/release/openshell-sandbox"), + docker_workspace_root().join("target/debug/openshell-sandbox"), ] { if candidate.is_file() { return canonicalize_existing_file(&candidate, "Docker host control supervisor"); @@ -4760,16 +4726,22 @@ async fn resolve_supervisor_runtime( fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { match daemon_arch { - "arm64" => vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )], - "amd64" => vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )], + "arm64" => vec![ + docker_workspace_root() + .join("target/aarch64-unknown-linux-gnu/release/openshell-sandbox"), + ], + "amd64" => vec![ + docker_workspace_root() + .join("target/x86_64-unknown-linux-gnu/release/openshell-sandbox"), + ], _ => Vec::new(), } } +fn docker_workspace_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + /// Pull the supervisor image (if not already local), extract /// `/openshell-sandbox` to a host cache keyed by the image's content /// digest, and return the cache path. diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 955c1542dc..22da62919f 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -97,13 +97,11 @@ fn runtime_config() -> DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), image_pull_policy: String::new(), sandbox_namespace: "default".to_string(), - grpc_endpoint: "https://localhost:8443".to_string(), gateway_route: DockerGatewayRoute::Bridge { bind_address: SocketAddr::new( IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), DEFAULT_SERVER_PORT, ), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), }, gateway_callback_bind_address: Some(SocketAddr::new( IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), @@ -251,14 +249,49 @@ fn docker_boundary_runtime_rejects_non_socket_stale_entry() { let socket = temporary.path().join("control.sock"); fs::write(&socket, b"not a socket").expect("write stale entry"); - let error = remove_owned_stale_boundary_socket(&socket) + let error = remove_owned_stale_socket(&socket, 0o666, true) .expect_err("non-socket stale entry must be rejected"); assert!( - error.contains("not an owned mode-0666 Unix socket"), + error.contains("not an expected owned mode-0666 Unix socket"), "{error}" ); } +#[cfg(unix)] +#[test] +fn docker_boundary_restart_removes_stale_control_and_ssh_sockets() { + use std::os::unix::fs::PermissionsExt as _; + use std::os::unix::net::UnixListener; + + let temporary = TempDir::new().expect("create temporary directory"); + let mut config = runtime_config(); + config.boundary_runtime_root = temporary.path().join("runtime"); + fs::create_dir(&config.boundary_runtime_root).expect("create runtime root"); + fs::set_permissions( + &config.boundary_runtime_root, + fs::Permissions::from_mode(0o700), + ) + .expect("restrict runtime root"); + let boundary = docker_boundary_mount_dir_by_id("sbx-123", &config); + fs::create_dir(&boundary).expect("create boundary directory"); + fs::set_permissions(&boundary, fs::Permissions::from_mode(0o700)) + .expect("restrict boundary directory"); + + let control = boundary.join("control.sock"); + let control_listener = UnixListener::bind(&control).expect("bind stale control socket"); + fs::set_permissions(&control, fs::Permissions::from_mode(0o666)).expect("set control mode"); + let ssh = docker_control_ssh_socket_by_id("sbx-123", &config); + let ssh_listener = UnixListener::bind(&ssh).expect("bind stale SSH socket"); + fs::set_permissions(&ssh, fs::Permissions::from_mode(0o600)).expect("set SSH mode"); + drop((control_listener, ssh_listener)); + + prepare_docker_boundary_restart_sockets("sbx-123", &config) + .expect("remove validated stale sockets"); + + assert!(!control.exists()); + assert!(!ssh.exists()); +} + #[tokio::test] async fn control_failure_overrides_running_container_readiness() { let driver = test_driver_with_config(runtime_config()); @@ -293,6 +326,30 @@ async fn control_failure_overrides_running_container_readiness() { assert!(ready.message.contains("control exited unexpectedly")); } +#[cfg(unix)] +#[test] +fn control_command_does_not_inherit_gateway_environment() { + let _guard = ENV_LOCK.lock().unwrap(); + temp_env::with_var( + "OPENSHELL_TEST_GATEWAY_SECRET", + Some("must-not-leak"), + || { + let runtime = tokio::runtime::Runtime::new().expect("create test runtime"); + let output = runtime + .block_on(async { + new_control_command(Path::new("/usr/bin/env")) + .output() + .await + }) + .expect("run environment probe"); + assert!(output.status.success()); + assert!( + !String::from_utf8_lossy(&output.stdout).contains("OPENSHELL_TEST_GATEWAY_SECRET") + ); + }, + ); +} + #[tokio::test] async fn tracing_in_process_service_preserves_the_driver_rpc_server_boundary() { use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; @@ -807,34 +864,6 @@ async fn host_gateway_route_reports_ipv4_loopback_callback_listener() { ); } -#[test] -fn container_visible_endpoint_rewrites_loopback_hosts() { - assert_eq!( - docker_container_openshell_endpoint( - "https://localhost:8443", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "https://host.openshell.internal:17670/" - ); - assert_eq!( - docker_container_openshell_endpoint( - "http://127.0.0.1:8080", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "http://host.openshell.internal:17670/" - ); - assert_eq!( - docker_container_openshell_endpoint( - "https://gateway.internal:8443", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "https://host.openshell.internal:17670/" - ); -} - #[test] fn docker_bridge_gateway_ip_requires_ipv4_gateway() { let network = bollard::models::NetworkInspect { @@ -899,13 +928,25 @@ fn docker_gateway_route_uses_host_gateway_for_docker_desktop() { ), DockerGatewayRoute::HostGateway ); - assert_eq!( - docker_extra_hosts(&DockerGatewayRoute::HostGateway), - vec![ - "host.docker.internal:host-gateway".to_string(), - "host.openshell.internal:host-gateway".to_string() - ] - ); +} + +#[test] +fn vm_backed_docker_daemon_rejects_shared_unix_boundary_transport() { + let desktop = SystemInfo { + operating_system: Some("Docker Desktop".to_string()), + ..Default::default() + }; + let native = SystemInfo { + operating_system: Some("Ubuntu 24.04".to_string()), + ..Default::default() + }; + + let error = validate_shared_unix_boundary_transport(&desktop, false) + .expect_err("VM-backed daemon must fail before provisioning"); + assert!(error.to_string().contains("shares host Unix sockets")); + validate_shared_unix_boundary_transport(&native, false) + .expect("local native daemon supports bind-mounted Unix sockets"); + assert!(validate_shared_unix_boundary_transport(&native, true).is_err()); } #[test] @@ -950,13 +991,6 @@ fn docker_gateway_route_uses_host_gateway_for_colima() { ), DockerGatewayRoute::HostGateway ); - assert_eq!( - docker_extra_hosts(&DockerGatewayRoute::HostGateway), - vec![ - "host.docker.internal:host-gateway".to_string(), - "host.openshell.internal:host-gateway".to_string() - ] - ); } #[test] @@ -1041,16 +1075,8 @@ fn docker_gateway_route_uses_bridge_gateway_for_linux_docker() { route, DockerGatewayRoute::Bridge { bind_address: "172.18.0.1:17670".parse().unwrap(), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), } ); - assert_eq!( - docker_extra_hosts(&route), - vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ] - ); } #[test] @@ -1090,16 +1116,8 @@ fn docker_gateway_route_prefers_configured_host_gateway_ip() { route, DockerGatewayRoute::Bridge { bind_address: "172.20.0.4:17670".parse().unwrap(), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4)), } ); - assert_eq!( - docker_extra_hosts(&route), - vec![ - "host.docker.internal:172.20.0.4".to_string(), - "host.openshell.internal:172.20.0.4".to_string() - ] - ); } #[test] @@ -1196,89 +1214,44 @@ fn container_create_body_sets_driver_owned_pids_limit() { } #[test] -fn build_environment_sets_docker_tls_paths() { - let env = build_environment(&test_sandbox(), &runtime_config()); - assert!(env.contains(&format!("OPENSHELL_TLS_CA={TLS_CA_MOUNT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_CERT={TLS_CERT_MOUNT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_KEY={TLS_KEY_MOUNT_PATH}"))); - assert!(env.contains(&"TEMPLATE_ENV=template".to_string())); - assert!(env.contains(&"SPEC_ENV=spec".to_string())); - assert!(env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - let encoded = env - .iter() - .find_map(|entry| { - entry - .strip_prefix("OPENSHELL_MAIN_PROCESS_SPEC=") - .map(str::to_string) - }) - .expect("main-process transport"); - let main = openshell_core::sandbox_env::MainProcessConfig::decode(&encoded).unwrap(); - assert_eq!(main.command, vec!["/bin/bash", "-l"]); - assert!(main.tty); -} - -#[test] -fn build_environment_keeps_network_capabilities_driver_controlled() { - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - "spoofed".to_string(), - ); - let env = build_environment(&sandbox, &runtime_config()); - assert!(env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); -} - -#[test] -fn build_environment_protects_oci_identity_metadata() { +fn docker_child_environment_strips_supervisor_control_keys() { let mut sandbox = test_sandbox(); let spec = sandbox.spec.as_mut().unwrap(); - for (key, value) in [ - (openshell_core::sandbox_env::OCI_IMAGE_USER, "spoofed"), - (openshell_core::sandbox_env::SANDBOX_UID, "9999"), - (openshell_core::sandbox_env::SANDBOX_GID, "9999"), + for key in [ + openshell_core::sandbox_env::ENDPOINT, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, ] { - spec.environment.insert(key.to_string(), value.to_string()); + spec.environment + .insert(key.to_string(), "spoofed".to_string()); } + spec.environment + .insert("PATH".to_string(), "/agent/bin".to_string()); - let env = build_environment_for_oci_user(&sandbox, &runtime_config(), "app:staff"); + let env = docker_child_environment(&sandbox); - assert!(env.contains(&format!( - "{}=app:staff", - openshell_core::sandbox_env::OCI_IMAGE_USER - ))); - assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_UID))); - assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_GID))); - assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); - assert!(!env.iter().any(|entry| entry.ends_with("=9999"))); + assert_eq!(env.get("PATH").map(String::as_str), Some("/agent/bin")); + assert!(env.contains_key("TEMPLATE_ENV")); + assert!(env.contains_key("SPEC_ENV")); + assert!(!env.values().any(|value| value == "spoofed")); } #[test] -fn build_environment_strips_gateway_tls_server_name() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.environment.insert( - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME.to_string(), - "evil.attacker.example.com".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); +fn boundary_environment_contains_only_driver_owned_values() { + let env = build_boundary_environment(&test_sandbox(), &runtime_config()); + assert_eq!(env.len(), 2); assert!( - !env.iter().any(|entry| entry.starts_with(&format!( - "{}=", - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME - ))), - "GATEWAY_TLS_SERVER_NAME must be stripped from the supervisor environment" + env.iter() + .any(|entry| entry.starts_with("OPENSHELL_LOG_LEVEL=")) ); + assert!(env.iter().any(|entry| entry.starts_with(&format!( + "{}=", + openshell_core::sandbox_env::TELEMETRY_ENABLED + )))); } #[test] @@ -1302,6 +1275,13 @@ fn container_creation_uses_inspected_immutable_image() { assert_eq!(body.image.as_deref(), Some("sha256:immutable")); assert_eq!(body.user.as_deref(), Some("0")); assert_eq!(body.working_dir.as_deref(), Some("/")); + assert_eq!( + body.labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_TOPOLOGY)) + .map(String::as_str), + Some(LABEL_ISOLATION_TOPOLOGY_BOUNDARY_V1) + ); assert_eq!( body.cmd.as_deref(), Some( @@ -1317,6 +1297,32 @@ fn container_creation_uses_inspected_immutable_image() { })); } +#[test] +fn boundary_control_requirement_recognizes_marker_and_existing_command() { + let labeled = ContainerSummary { + labels: Some(HashMap::from([( + LABEL_ISOLATION_TOPOLOGY.to_string(), + LABEL_ISOLATION_TOPOLOGY_BOUNDARY_V1.to_string(), + )])), + ..Default::default() + }; + let pre_marker = ContainerSummary { + command: Some( + "/usr/local/bin/openshell-sandbox --mode=boundary --boundary-config /run/config.json" + .to_string(), + ), + ..Default::default() + }; + let legacy = ContainerSummary { + command: Some("/usr/local/bin/openshell-sandbox".to_string()), + ..Default::default() + }; + + assert!(container_requires_boundary_control(&labeled)); + assert!(container_requires_boundary_control(&pre_marker)); + assert!(!container_requires_boundary_control(&legacy)); +} + #[test] fn container_creation_rejects_invalid_oci_working_dir() { let metadata = DockerImageMetadata { @@ -1483,64 +1489,6 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( .expect("/sandbox remains mountable when the inspected workspace is elsewhere"); } -#[test] -fn build_environment_keeps_path_driver_controlled() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.environment - .insert("PATH".to_string(), "/malicious/spec/bin".to_string()); - spec.template - .as_mut() - .unwrap() - .environment - .insert("PATH".to_string(), "/malicious/template/bin".to_string()); - - let env = build_environment(&sandbox, &runtime_config()); - let path_entries = env - .iter() - .filter(|entry| entry.starts_with("PATH=")) - .collect::>(); - - let expected_path = format!("PATH={SUPERVISOR_PATH}"); - assert_eq!(path_entries.len(), 1); - assert_eq!(path_entries[0], &expected_path); -} - -#[test] -fn build_environment_keeps_telemetry_toggle_driver_controlled() { - let _guard = ENV_LOCK.lock().unwrap(); - temp_env::with_vars( - [( - openshell_core::sandbox_env::TELEMETRY_ENABLED, - Some("false"), - )], - || { - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - "true".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); - let telemetry_entries = env - .iter() - .filter(|entry| { - entry.starts_with(&format!( - "{}=", - openshell_core::sandbox_env::TELEMETRY_ENABLED - )) - }) - .collect::>(); - - assert_eq!(telemetry_entries.len(), 1); - assert_eq!( - telemetry_entries[0], - &format!("{}=false", openshell_core::sandbox_env::TELEMETRY_ENABLED) - ); - }, - ); -} - #[test] fn build_binds_exposes_only_boundary_runtime_material() { let binds = build_binds(&test_sandbox(), &runtime_config()); @@ -2109,27 +2057,6 @@ fn docker_nonlocal_volume_with_bind_option_is_not_bind_backed() { assert!(!docker_volume_is_bind_backed(&volume)); } -#[test] -fn build_environment_uses_token_file_without_raw_token_env() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.sandbox_token = "secret.jwt.value".to_string(); - spec.environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN.to_string(), - "user-provided-token".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); - - assert!(!env.iter().any(|entry| { - entry.starts_with(&format!("{}=", openshell_core::sandbox_env::SANDBOX_TOKEN)) - })); - assert!(env.contains(&format!( - "{}={SANDBOX_TOKEN_MOUNT_PATH}", - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE - ))); -} - #[test] fn managed_container_label_filters_include_gateway_namespace() { let filters = @@ -2868,15 +2795,21 @@ fn docker_guest_tls_paths_require_all_files_for_https() { fn linux_supervisor_candidates_follow_daemon_arch() { assert_eq!( linux_supervisor_candidates("amd64"), - vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )] + vec![ + docker_workspace_root() + .join("target/x86_64-unknown-linux-gnu/release/openshell-sandbox") + ] ); assert_eq!( linux_supervisor_candidates("arm64"), - vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )] + vec![ + docker_workspace_root() + .join("target/aarch64-unknown-linux-gnu/release/openshell-sandbox") + ] + ); + assert!( + linux_supervisor_candidates("amd64")[0].is_absolute(), + "developer fallback must never resolve relative to the gateway cwd" ); } From 8780abe5b07c093041f0820e1d8895eadbfcea17 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 02:46:45 -0700 Subject: [PATCH 5/6] fix(docker): preserve host aliases without networking Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/src/lib.rs | 9 +++++++++ crates/openshell-driver-docker/src/tests.rs | 9 ++++++++- e2e/rust/tests/forward_proxy_l7_bypass.rs | 16 ++++++++-------- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 2fe5162f07..ae22715cdf 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -3927,6 +3927,15 @@ fn build_container_create_body_for_image( // carries each accepted stream to host control over the private // Unix socket. network_mode: Some("none".to_string()), + // `network_mode=none` disables Docker's embedded DNS, but the + // stable host aliases are still part of the driver contract. Ask + // Docker to materialize them in `/etc/hosts`; workload traffic + // remains fenced to the boundary proxy, while libc lookups and + // proxy clients retain the portable hostname identity. + extra_hosts: Some(vec![ + format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), + format!("{HOST_DOCKER_INTERNAL}:host-gateway"), + ]), ..Default::default() }), networking_config: None, diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 22da62919f..67534d8a71 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -2560,7 +2560,14 @@ fn build_container_create_body_disables_docker_networking() { Some("none".to_string()), "only the nested workload namespace should receive mediated egress" ); - assert!(host_config.extra_hosts.is_none()); + assert_eq!( + host_config.extra_hosts, + Some(vec![ + "host.openshell.internal:host-gateway".to_string(), + "host.docker.internal:host-gateway".to_string(), + ]), + "networkless workloads still need stable host aliases in /etc/hosts" + ); } #[test] diff --git a/e2e/rust/tests/forward_proxy_l7_bypass.rs b/e2e/rust/tests/forward_proxy_l7_bypass.rs index f5df4f53e3..29261ff635 100644 --- a/e2e/rust/tests/forward_proxy_l7_bypass.rs +++ b/e2e/rust/tests/forward_proxy_l7_bypass.rs @@ -10,13 +10,13 @@ use std::io::Write; -use openshell_e2e::harness::container::ContainerHttpServer; +use openshell_e2e::harness::container::HostSupportContainer; use openshell_e2e::harness::sandbox::SandboxGuard; use tempfile::NamedTempFile; -const TEST_SERVER_ALIAS: &str = "rest-l7.openshell.test"; +const TEST_SERVER_HOST: &str = "host.openshell.internal"; -async fn start_test_server() -> Result { +async fn start_test_server() -> Result { let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer class Handler(BaseHTTPRequestHandler): @@ -34,7 +34,7 @@ class Handler(BaseHTTPRequestHandler): HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() "#; - ContainerHttpServer::start_python(TEST_SERVER_ALIAS, script).await + HostSupportContainer::start_python(script, 8000).await } fn write_policy_with_l7_rules(host: &str, port: u16) -> Result { @@ -100,7 +100,7 @@ network_policies: async fn forward_proxy_allows_l7_permitted_request() { let server = start_test_server().await.expect("start test server"); let policy = - write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); + write_policy_with_l7_rules(TEST_SERVER_HOST, server.port).expect("write custom policy"); let policy_path = policy .path() .to_str() @@ -129,7 +129,7 @@ for attempt in range(6): break print(json.dumps(last)) "#, - host = server.host, + host = TEST_SERVER_HOST, port = server.port, ); @@ -150,7 +150,7 @@ print(json.dumps(last)) async fn forward_proxy_denies_l7_blocked_request() { let server = start_test_server().await.expect("start test server"); let policy = - write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); + write_policy_with_l7_rules(TEST_SERVER_HOST, server.port).expect("write custom policy"); let policy_path = policy .path() .to_str() @@ -170,7 +170,7 @@ except urllib.error.HTTPError as e: except Exception as e: print(json.dumps({{"status": -1, "error": str(e)}})) "#, - host = server.host, + host = TEST_SERVER_HOST, port = server.port, ); From c63a6043220e3f2420ac6d980674c19d14fb0019 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 3 Sep 2026 06:23:18 -0700 Subject: [PATCH 6/6] test(docker): assert networkless host aliases Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/src/tests.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 67534d8a71..bdfe961e2f 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -2101,7 +2101,14 @@ fn build_container_create_body_replaces_inherited_cmd_with_boundary_mode() { Some(&vec!["apparmor=unconfined".to_string()]) ); assert_eq!(host_config.network_mode.as_deref(), Some("none")); - assert!(host_config.extra_hosts.is_none()); + assert_eq!( + host_config.extra_hosts, + Some(vec![ + "host.openshell.internal:host-gateway".to_string(), + "host.docker.internal:host-gateway".to_string(), + ]), + "networkless workloads still receive stable host aliases" + ); assert!(create_body.networking_config.is_none()); }