diff --git a/Cargo.lock b/Cargo.lock index 2a614f6015..0a90c0fc30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4071,6 +4071,7 @@ dependencies = [ name = "openshell-driver-vm" version = "0.0.0" dependencies = [ + "base64 0.22.1", "bollard", "clap", "flate2", @@ -4083,6 +4084,7 @@ dependencies = [ "oci-client", "openshell-core", "openshell-driver-podman", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", "openshell-policy", @@ -4092,6 +4094,7 @@ dependencies = [ "polling", "prost", "prost-types", + "rand 0.9.4", "rustix 1.1.4", "serde", "serde_json", @@ -4179,14 +4182,14 @@ dependencies = [ "libc", "openshell-core", "rcgen", - "rustls 0.23.38", + "rustls", "rustls-pemfile", "serde", "serde_json", - "socket2 0.6.3", + "socket2", "thiserror 2.0.18", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", ] [[package]] @@ -4315,7 +4318,7 @@ dependencies = [ "temp-env", "tempfile", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tokio-tungstenite 0.26.2", "tonic", "tracing", diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index 0d250b6e71..b46d4b9049 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -17,45 +17,83 @@ path = "src/lib.rs" [[bin]] name = "openshell-driver-vm" path = "src/main.rs" +required-features = ["compute-driver"] [dependencies] openshell-core = { path = "../openshell-core", default-features = false } -openshell-otel = { path = "../openshell-otel" } -openshell-policy = { path = "../openshell-policy" } -openshell-driver-podman = { path = "../openshell-driver-podman" } -openshell-vfio = { path = "../openshell-vfio" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } +openshell-otel = { path = "../openshell-otel", optional = true } +openshell-policy = { path = "../openshell-policy", optional = true } +openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } +openshell-vfio = { path = "../openshell-vfio", optional = true } -bollard = { version = "0.20", features = ["ssh"] } +base64 = { workspace = true, optional = true } +bollard = { version = "0.20", features = ["ssh"], optional = true } tokio = { workspace = true } -tonic = { workspace = true, features = ["transport"] } -tower-http = { workspace = true } -http = { workspace = true } -prost = { workspace = true } -prost-types = { workspace = true } -futures = { workspace = true } -tokio-stream = { workspace = true, features = ["net"] } -nix = { workspace = true } -clap = { workspace = true } -tracing = { workspace = true } -tracing-subscriber = { workspace = true } -opentelemetry = { workspace = true } -opentelemetry_sdk = { workspace = true } -tracing-opentelemetry = { workspace = true } -miette = { workspace = true } -url = { workspace = true } +tonic = { workspace = true, features = ["transport"], optional = true } +tower-http = { workspace = true, optional = true } +http = { workspace = true, optional = true } +prost = { workspace = true, optional = true } +prost-types = { workspace = true, optional = true } +futures = { workspace = true, optional = true } +tokio-stream = { workspace = true, features = ["net"], optional = true } +nix = { workspace = true, optional = true } +clap = { workspace = true, optional = true } +tracing = { workspace = true, optional = true } +tracing-subscriber = { workspace = true, optional = true } +opentelemetry = { workspace = true, optional = true } +opentelemetry_sdk = { workspace = true, optional = true } +tracing-opentelemetry = { workspace = true, optional = true } +miette = { workspace = true, optional = true } +rand = { workspace = true, optional = true } +url = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true } -oci-client = "0.16" +oci-client = { version = "0.16", optional = true } libc = "0.2" -rustix = { workspace = true } -libloading = "0.8" -tar = "0.4" -flate2 = "1" -sha2 = "0.10" -zstd = "0.13" +rustix = { workspace = true, optional = true } +libloading = { version = "0.8", optional = true } +tar = { version = "0.4", optional = true } +flate2 = { version = "1", optional = true } +sha2 = { version = "0.10", optional = true } +zstd = { version = "0.13", optional = true } [features] -default = ["telemetry"] +default = ["compute-driver", "telemetry"] +## Build the standalone compute driver and its host runtime implementation. +compute-driver = [ + "dep:base64", + "dep:bollard", + "dep:clap", + "dep:flate2", + "dep:futures", + "dep:http", + "dep:libloading", + "dep:miette", + "dep:nix", + "dep:oci-client", + "dep:openshell-otel", + "dep:openshell-policy", + "dep:openshell-driver-podman", + "dep:openshell-vfio", + "dep:opentelemetry", + "dep:opentelemetry_sdk", + "dep:polling", + "dep:prost", + "dep:prost-types", + "dep:rand", + "dep:rustix", + "dep:sha2", + "dep:tar", + "dep:tokio-stream", + "dep:tonic", + "dep:tower-http", + "dep:tracing", + "dep:tracing-opentelemetry", + "dep:tracing-subscriber", + "dep:url", + "dep:zstd", +] ## Compile in telemetry support (forwards to openshell-core/telemetry). On by ## default; build with `--no-default-features` for a telemetry-free VM driver ## that reports telemetry disabled to the sandboxes it launches. @@ -68,7 +106,7 @@ telemetry = ["openshell-core/telemetry"] ## enabling it alongside `telemetry` is a compile error rather than a silent ## telemetry-on build. Kept in sync with `default` by ## `rust:verify:defaults-without-telemetry`. -defaults-without-telemetry = [] +defaults-without-telemetry = ["compute-driver"] [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } @@ -82,7 +120,7 @@ opentelemetry_sdk = { workspace = true, features = ["testing"] } # nix::sys::prctl::set_pdeathsig there keeps the Linux path a single # syscall with no helper thread. [target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly"))'.dependencies] -polling = "3.11" +polling = { version = "3.11", optional = true } [lints] workspace = true diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 5c61ae1823..d45f252fe1 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -1,33 +1,40 @@ # openshell-driver-vm -> Status: Experimental. The VM compute driver is under active development and the interface still has VM-specific plumbing that will be generalized. +> Status: Experimental. The VM compute driver is under active development. -Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess, talks to it over a Unix domain socket with the `openshell.compute.v1.ComputeDriver` gRPC surface, and lets it manage per-sandbox microVMs. The runtime (libkrun + libkrunfw + gvproxy), guest OCI unpacker, and sandbox supervisor are embedded directly in the binary; each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir. +Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess and talks to it over the `openshell.compute.v1.ComputeDriver` Unix-socket surface. `openshell-sandbox --mode=control` runs as a native host process. The same binary runs as `--mode=boundary` inside each microVM and applies guest-local isolation through the shared RFC 0012 boundary protocol over virtio-vsock. + +The driver still embeds libkrun, libkrunfw, gvproxy, the guest OCI unpacker, the portable guest leaf, and the existing custom kernel runtime. Each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir. ## How it fits together ```mermaid flowchart LR - subgraph host["Host process"] + subgraph host["Host"] gateway["openshell-gateway
(vm::spawn)"] - driver["openshell-driver-vm
├── libkrun (VM)
├── gvproxy (net)
└── openshell-sandbox.zst"] + driver["openshell-driver-vm
libkrun + gvproxy"] + supervisor["openshell-sandbox
logical supervisor"] gateway <-->|"gRPC over UDS
compute-driver.sock"| driver + supervisor <-->|"authenticated gRPC
policy + relay"| gateway end subgraph guest["Per-sandbox microVM"] init["/srv/openshell-vm-
sandbox-init.sh"] - supervisor["/opt/openshell/bin/
openshell-sandbox
(PID 1)"] - init --> supervisor + leaf["openshell-sandbox --mode=boundary
boundary role (PID 1)"] + workload["sandbox workload"] + init --> leaf --> workload end driver -->|"CreateSandbox
boots via libkrun"| guest - supervisor -.->|"gRPC callback
--grpc-endpoint"| gateway + supervisor <-->|"authenticated RFC 0012
over virtio-vsock"| leaf - client["openshell-cli"] -->|"SSH proxy
127.0.0.1:<port>"| supervisor + client["openshell-cli"] -->|"connect / exec / forward"| gateway client -->|"CreateSandbox / Watch"| gateway ``` -Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside the VM. gvproxy exposes a single inbound SSH port (`host:` → `guest:2222`) and provides virtio-net egress. +The control role owns gateway credentials, admitted policy, provider resolution, middleware, the network proxy, and relay registration. The boundary role receives no gateway JWT or mTLS key. It receives a driver-authored boundary token and only the policy/environment state needed to launch the workload. + +VM-specific RFC 0012 code under `src/isolation/` only chooses the vsock transport and binds immutable VM generation and image claims into the protected guest config and host descriptor. Lifecycle, authentication, process control, binary identity, forwarding, and streaming come from `openshell-isolation-interface` and `openshell-sandbox`. ## Quick start (recommended) @@ -35,7 +42,7 @@ Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside th mise run gateway:vm ``` -First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the bundled guest supervisor. Subsequent runs are cached. +First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the portable Linux guest leaf plus its trusted helper runtime. The development task also builds the native host supervisor. Subsequent runs are cached. By default `mise run gateway:vm`: @@ -96,13 +103,13 @@ rm -rf "${XDG_CONFIG_HOME:-$HOME/.config}/openshell/gateways/vm-dev" If you want to drive the launch yourself instead of using `mise run gateway:vm` (i.e. `tasks/scripts/gateway-vm.sh`): ```shell -# 1. Stage runtime artifacts + supervisor bundle into target/vm-runtime-compressed/ +# 1. Stage runtime artifacts + guest process leaf into target/vm-runtime-compressed/ mise run vm:setup -mise run vm:supervisor # if openshell-sandbox.zst is not already present +mise run vm:supervisor # builds the Linux guest leaf and trusted helper runtime -# 2. Build both binaries with the staged artifacts embedded +# 2. Build gateway, native host supervisor, and driver OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ - cargo build -p openshell-gateway -p openshell-driver-vm + cargo build -p openshell-gateway -p openshell-sandbox -p openshell-driver-vm # 3. macOS only: codesign the driver for Hypervisor.framework codesign \ @@ -121,7 +128,7 @@ disable_tls = true [openshell.drivers.vm] default_image = "" -grpc_endpoint = "http://host.containers.internal:18081" +grpc_endpoint = "http://127.0.0.1:18081" driver_dir = "$PWD/target/debug" state_dir = "/tmp/openshell-vm-driver-$USER-vm-dev" EOF @@ -142,8 +149,8 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr | Configuration key | Default | Purpose | |---|---|---| -| `grpc_endpoint` | empty | Required. URL the sandbox guest dials to reach the gateway. Use `http://host.containers.internal:` (or `host.docker.internal` / `host.openshell.internal`) so traffic flows through gvproxy's host-loopback NAT (HostIP `192.168.127.254` → host `127.0.0.1`). Loopback URLs like `http://127.0.0.1:` are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | -| `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. | +| `grpc_endpoint` | empty | Required. URL the native host supervisor uses to reach the gateway. Host loopback such as `http://127.0.0.1:` is valid. Legacy guest aliases are normalized to host loopback. This endpoint is never sent into the VM. | +| `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. Relative paths are resolved to absolute paths at driver startup. | | `driver_dir` | unset | Override the directory searched for `openshell-driver-vm`. | | `default_image` | OpenShell base image | Sandbox image used when a create request omits one. | | `bootstrap_image` | unset | VM runtime image used as the immutable bootstrap root disk. Defaults to the sandbox image when unset. | @@ -151,17 +158,17 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `guest_tls_ca` | unset | CA cert for the guest's mTLS client bundle. Required when `grpc_endpoint` uses `https://`. | -| `guest_tls_cert` | unset | Guest client certificate. | -| `guest_tls_key` | unset | Guest client private key. | -| `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) the in-guest supervisor chains policy-approved TLS CONNECT egress through. On the libkrun backend a proxy on the gateway host's loopback must be addressed as `http://host.openshell.internal:` — guest egress leaves through gvproxy, which NATs `192.168.127.254` to the host's `127.0.0.1`. The QEMU/TAP backend (GPU sandboxes) has no such NAT and its nftables rules expose only the gateway port to the guest, so a gateway-host proxy URL is rejected at launch there; use an address routable from the guest's masqueraded egress. | +| `guest_tls_ca` | unset | Historical key name for the host supervisor's gateway CA certificate. Required when `grpc_endpoint` uses `https://`; never copied into the guest. | +| `guest_tls_cert` | unset | Historical key name for the host supervisor's client certificate; never copied into the guest. | +| `guest_tls_key` | unset | Historical key name for the host supervisor's client private key; never copied into the guest. | +| `https_proxy` | unset | Corporate forward proxy (`http://host:port` or `https://host:port`) that host control chains policy-approved TLS CONNECT egress through. Host-loopback proxy URLs work because control runs on the gateway host. | | `no_proxy` | unset | Comma-separated bypass list for the corporate proxy only. OpenShell policy evaluation still applies. | | `proxy_auth_file` | unset | Gateway-host path to a `user:pass` credential file. Staged root-only into the per-sandbox overlay and removed with the sandbox. | | `proxy_auth_allow_insecure` | unset | Required with `proxy_auth_file` against an `http://` proxy: acknowledges that Basic auth is cleartext on the connection to the proxy. | | `proxy_connect_by_hostname` | unset | Send hostnames rather than validated IPs in CONNECT. Last resort for proxies whose ACLs reject IP targets. | | `proxy_ca_bundle` | unset | Gateway-host path to a PEM CA bundle trusted for an `https://` proxy and for certificates a TLS-intercepting proxy re-signs. | -The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and they reach the supervisor on its command line through a per-sandbox argument file the driver writes into the overlay upperdir on every launch, so a sandbox image cannot forge or shadow them. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial. +The proxy settings are operator-owned and deployment-level: they are not accepted through `template.driver_config.vm`, and the driver passes them only to native host control. Every present-but-invalid value is fatal at gateway or sandbox startup rather than degrading to a direct dial. See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the gateway process flag surface. @@ -219,9 +226,16 @@ marked sandboxes without launching compute. Start removes the marker and uses the normal persisted restore path with the existing overlay. Delete removes the entire sandbox state directory, including a stop marker and overlay. -The driver records a terminal tombstone when the canonical main process exits. -Driver startup reports that sandbox as terminal instead of relaunching the VM, -even when the process exited successfully. +The host control writes and syncs a terminal tombstone when the canonical main +process exits, before it reports completion and while it retains the boundary +for exec and forwarding. Driver startup reports that sandbox as terminal +instead of relaunching the VM, even when the process exited successfully. + +When the packaged host supervisor is not installed beside the driver, the +driver extracts its embedded copy into `/host-runtime`. It accepts a +cached binary only when its SHA-256 content matches the embedded supervisor and +it remains an executable regular file. Replacement is written and synced under +a temporary name, then atomically renamed into place. ## Logs and debugging @@ -233,14 +247,17 @@ RUST_LOG=openshell_server=debug,openshell_driver_vm=debug \ ``` The VM guest's serial console is appended to `//console.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions and removes same-owner stale sockets. On clean shutdown, the gateway sends the managed driver `SIGTERM`, waits up to five seconds for it to flush telemetry and exit, then force-kills it if necessary and removes the socket. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development. +The VM serial console is appended to `/sandboxes//rootfs-console.log`. Host-supervisor stdout and stderr are written beside it as `supervisor.log` and `supervisor.err.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions, removes same-owner stale sockets, and the gateway removes the socket on clean shutdown via `ManagedDriverProcess::drop`. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development. ## Host-side nftables rules -The VM driver creates a per-VM nftables table on the host (`openshell_vm_vmtap_`) with three chains. These rules serve two purposes: NAT infrastructure (required for VM connectivity) and defense-in-depth host isolation. Primary security enforcement — proxy-only egress and bypass detection — is handled by the sandbox supervisor's own nftables rules inside the VM guest. +This section applies to the QEMU/VFIO path, which uses a host TAP device. The normal libkrun path uses gvproxy and needs KVM access but does not require `CAP_NET_ADMIN`. The host supervisor performs primary policy enforcement and receives guest-originated connections through the authenticated network-mediation stream. + +The QEMU path creates a per-VM nftables table on the host (`openshell_vm_vmtap_`) with three chains for NAT infrastructure and defense-in-depth host isolation. **`postrouting` (NAT):** Masquerades outbound VM traffic so it can be routed from the VM's private subnet to the external network. This chain handles forwarded traffic (VM → internet), not traffic destined for the host. -**`forward` (defense-in-depth):** Accepts all outbound traffic from the VM (security enforcement happens guest-side) and accepts established/related response traffic back to the VM. Drops unsolicited inbound connections to the VM from the broader network. This chain handles forwarded traffic only — packets transiting the host between the TAP interface and other interfaces. +**`forward` (defense-in-depth):** Accepts outbound traffic from the VM and established/related response traffic back to the VM. Drops unsolicited inbound connections to the VM from the broader network. This chain handles forwarded traffic only — packets transiting the host between the TAP interface and other interfaces. **`input` (defense-in-depth):** Accepts traffic from the VM to the gateway port on the host. Drops all other traffic from the VM destined for the host itself. This limits what a compromised guest can reach on the host to the gateway service only. @@ -258,9 +275,9 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical - macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM - Rust toolchain - e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection -- Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch): +- Guest-leaf cross-compile toolchain (needed on macOS, and on Linux when host arch differs from the guest): - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest) - - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary. + - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the Linux `openshell-sandbox --mode=boundary` binary. - [mise](https://mise.jdx.dev/) task runner - Docker or Podman socket on the local CLI/gateway host when using `openshell sandbox create --from ./Dockerfile` or `--from ./dir`; the CLI @@ -291,11 +308,12 @@ The RPM gateway package is configured for the Podman driver. On Apple Silicon macOS, `install.sh` stages the generated `openshell.rb` formula from the selected release in the `nvidia/openshell` Homebrew tap. -Homebrew installs `openshell`, `openshell-gateway`, and -`openshell-driver-vm`, ad-hoc signs the driver with the Hypervisor entitlement -in `post_install`, and owns the `brew services` gateway lifecycle. The service -also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless -the user explicitly overrides it. +Homebrew installs `openshell`, `openshell-gateway`, `openshell-driver-vm`, and +the native `openshell-sandbox` host supervisor beside the driver. It ad-hoc +signs the driver with the Hypervisor entitlement in `post_install` and owns the +`brew services` gateway lifecycle. The service also leaves `OPENSHELL_DRIVERS` +unset so driver choice remains automatic unless the user explicitly overrides +it. ## TODOs diff --git a/crates/openshell-driver-vm/build.rs b/crates/openshell-driver-vm/build.rs index 92532ed7b2..5d0f748f91 100644 --- a/crates/openshell-driver-vm/build.rs +++ b/crates/openshell-driver-vm/build.rs @@ -10,6 +10,10 @@ use std::path::{Path, PathBuf}; use std::{env, fs}; fn main() { + if env::var_os("CARGO_FEATURE_COMPUTE_DRIVER").is_none() { + return; + } + println!("cargo:rerun-if-env-changed=OPENSHELL_VM_RUNTIME_COMPRESSED_DIR"); if let Ok(dir) = env::var("OPENSHELL_VM_RUNTIME_COMPRESSED_DIR") { @@ -38,7 +42,14 @@ fn main() { println!("cargo:warning=VM runtime not available for {target_os}-{target_arch}"); generate_stub_resources( &out_dir, - &["libkrun", "libkrunfw", "openshell-sandbox.zst", "umoci.zst"], + &[ + "libkrun", + "libkrunfw", + "gvproxy.zst", + "openshell-sandbox.zst", + "openshell-runtime.tar.zst", + "umoci.zst", + ], ); return; } @@ -56,6 +67,7 @@ fn main() { &format!("{libkrunfw_name}.zst"), "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ], ); @@ -79,6 +91,10 @@ fn main() { "openshell-sandbox.zst".to_string(), "openshell-sandbox.zst".to_string(), ), + ( + "openshell-runtime.tar.zst".to_string(), + "openshell-runtime.tar.zst".to_string(), + ), ("umoci.zst".to_string(), "umoci.zst".to_string()), ]; diff --git a/crates/openshell-driver-vm/runtime/README.md b/crates/openshell-driver-vm/runtime/README.md index b686874ba2..35ffe8e408 100644 --- a/crates/openshell-driver-vm/runtime/README.md +++ b/crates/openshell-driver-vm/runtime/README.md @@ -12,14 +12,18 @@ runtime/ ``` `openshell-driver-vm` embeds libkrun, libkrunfw, gvproxy, umoci for guest-side -OCI image unpacking, and the bundled `openshell-sandbox` supervisor. +OCI image unpacking, and the portable `openshell-sandbox --mode=boundary` role. +VMs do not attach a guest NIC. The boundary carries control, mediated network, +and DNS streams over the authenticated vsock channel; gvproxy remains in the +runtime bundle for artifact compatibility but is not started. ## Why The stock `libkrunfw` kernel does not include the bridge, netfilter, -conntrack, cgroup, seccomp, and Landlock features the sandbox supervisor needs -inside each microVM. `kernel/openshell.kconfig` extends the libkrunfw kernel so -VM sandboxes can run the same supervisor enforcement path as other backends. +conntrack, cgroup, seccomp, and Landlock features the process leaf needs inside +each microVM. `kernel/openshell.kconfig` extends the libkrunfw kernel so VM +sandboxes retain guest-local process and filesystem enforcement while the +logical supervisor runs on the host. ## Build Scripts @@ -36,12 +40,12 @@ VM sandboxes can run the same supervisor enforcement path as other backends. # Download the current pre-built runtime and stage compressed artifacts mise run vm:setup -# Build the bundled guest supervisor +# Build the portable Linux guest leaf and trusted helper runtime (requires Docker Buildx) mise run vm:supervisor -# Build the gateway and VM driver with embedded runtime artifacts +# Build the gateway, native host supervisor, and VM driver OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ - cargo build -p openshell-gateway -p openshell-driver-vm + cargo build -p openshell-gateway -p openshell-sandbox -p openshell-driver-vm ``` Use `FROM_SOURCE=1 mise run vm:setup` to build the runtime from source instead diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 32d6ed1dff..854176a69b 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # Minimal init for sandbox VMs. Runs as PID 1 inside the guest, mounts the -# essential filesystems, configures networking (gvproxy DHCP or TAP static), -# optionally loads NVIDIA GPU drivers, then execs the OpenShell sandbox -# supervisor. +# essential filesystems, optionally loads NVIDIA GPU drivers, then execs the +# portable VM process leaf. Workload networking crosses the authenticated +# boundary channel; the VM does not receive a network interface. set -euo pipefail @@ -14,27 +14,9 @@ set -euo pipefail unset KRUN_INIT_PID1 BOOT_START=$(date +%s%3N 2>/dev/null || date +%s) -# gvisor-tap-vsock subnet layout: -# 192.168.127.1 — gateway: gvproxy's DNS / DHCP / HTTP API. Does NOT -# proxy arbitrary host ports. -# 192.168.127.254 — host-loopback: NAT-rewritten to host's 127.0.0.1 by -# gvproxy's TCP/UDP/ICMP forwarder. Use this address -# (or any of the host.* hostnames below) to reach a -# service the host is listening on. -# The host.openshell.internal / host.containers.internal / -# host.docker.internal DNS records served by gvproxy's embedded resolver -# point at 192.168.127.254. We mirror that in /etc/hosts so the supervisor -# can reach the gateway even when gvproxy's DNS is not in resolv.conf -# (e.g. DHCP failed and we fell back to 8.8.8.8). -GVPROXY_GATEWAY_IP="192.168.127.1" -GVPROXY_HOST_LOOPBACK_IP="192.168.127.254" -GATEWAY_IP="$GVPROXY_GATEWAY_IP" SANDBOX_OWNER_NORMALIZED_MARKER="/opt/openshell/.sandbox-owner-normalized" GPU_ENABLED="${GPU_ENABLED:-false}" -VM_NET_IP="${VM_NET_IP:-}" -VM_NET_GW="${VM_NET_GW:-}" -VM_NET_DNS="${VM_NET_DNS:-}" ts() { local now @@ -117,6 +99,10 @@ ensure_target_runtime() { cp /opt/openshell/bin/openshell-sandbox "$image_root/opt/openshell/bin/openshell-sandbox" chmod 0755 "$image_root/opt/openshell/bin/openshell-sandbox" fi + if [ -d /opt/openshell/bin/openshell-runtime ]; then + rm -rf "$image_root/opt/openshell/bin/openshell-runtime" + cp -a /opt/openshell/bin/openshell-runtime "$image_root/opt/openshell/bin/openshell-runtime" + fi touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow" if ! grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then @@ -275,16 +261,17 @@ exec_supervisor_in_newroot() { "${bootstrap}/lib64/ld-linux-aarch64.so.1"; do if [ -x "/newroot${loader}" ]; then lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu" - exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" \ - "$supervisor" --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" + exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" \ + --mode=boundary --boundary-config /etc/openshell/boundary.json fi done - exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" + exec "$chroot_bin" /newroot "$supervisor" \ + --mode=boundary --boundary-config /etc/openshell/boundary.json fi if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox \ - --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" + --mode=boundary --boundary-config /etc/openshell/boundary.json fi done @@ -355,170 +342,6 @@ setup_overlay_root() { run_post_overlay_setup } -parse_endpoint() { - local endpoint="$1" - local scheme rest authority path host port - - case "$endpoint" in - *://*) - scheme="${endpoint%%://*}" - rest="${endpoint#*://}" - ;; - *) - return 1 - ;; - esac - - authority="${rest%%/*}" - path="${rest#"$authority"}" - if [ "$path" = "$rest" ]; then - path="" - fi - - if [[ "$authority" =~ ^\[([^]]+)\]:(.+)$ ]]; then - host="${BASH_REMATCH[1]}" - port="${BASH_REMATCH[2]}" - elif [[ "$authority" =~ ^\[([^]]+)\]$ ]]; then - host="${BASH_REMATCH[1]}" - port="" - elif [[ "$authority" == *:* ]]; then - host="${authority%%:*}" - port="${authority##*:}" - else - host="$authority" - port="" - fi - - if [ -z "$port" ]; then - case "$scheme" in - https) port="443" ;; - *) port="80" ;; - esac - fi - - printf '%s\n%s\n%s\n%s\n' "$scheme" "$host" "$port" "$path" -} - -tcp_probe() { - local host="$1" - local port="$2" - - if command -v timeout >/dev/null 2>&1; then - timeout 2 bash -c "exec 3<>/dev/tcp/\$1/\$2" _ "$host" "$port" >/dev/null 2>&1 - else - bash -c "exec 3<>/dev/tcp/\$1/\$2" _ "$host" "$port" >/dev/null 2>&1 - fi -} - -ensure_host_gateway_aliases() { - # Seed /etc/hosts with the well-known gvproxy hostnames so the supervisor - # can reach the OpenShell server even when gvproxy's built-in DNS is not - # in resolv.conf (e.g. when DHCP fails and we fall back to 8.8.8.8). - # - # Critical distinction: host.* aliases point at the gvproxy *host-loopback* - # IP (192.168.127.254), not the gateway IP (192.168.127.1). Only the - # host-loopback IP carries NAT rewriting to the host's 127.0.0.1 — the - # gateway IP only listens on gvproxy's own service ports (DNS:53, DHCP, - # HTTP API:80). Pinning host.containers.internal to the gateway IP - # silently breaks guest→host port reachability for arbitrary ports. - local host_aliases="host.openshell.internal host.containers.internal host.docker.internal" - local gateway_aliases="gateway.containers.internal" - local filter='(^|[[:space:]])(host\.openshell\.internal|host\.containers\.internal|host\.docker\.internal|gateway\.containers\.internal)([[:space:]]|$)' - - write_host_gateway_aliases "$(root_path /etc/hosts)" "$(root_path "/tmp/openshell-hosts.$$.tmp")" || true - if [ -n "${ROOT_PREFIX:-}" ]; then - write_host_gateway_aliases "/etc/hosts" "/tmp/openshell-hosts.$$.tmp" || true - fi -} - -write_host_gateway_aliases() { - local hosts_path="$1" - local hosts_tmp="$2" - mkdir -p "$(dirname "$hosts_path")" 2>/dev/null || true - mkdir -p "$(dirname "$hosts_tmp")" 2>/dev/null || true - if [ -f "$hosts_path" ]; then - grep -vE "$filter" "$hosts_path" > "$hosts_tmp" || true - else - : > "$hosts_tmp" - fi - - # In TAP/GPU mode, GATEWAY_IP is overridden to VM_NET_GW (the host-side - # of the TAP), and the gateway is reachable directly there. In gvproxy - # mode, host.openshell.internal etc. need GVPROXY_HOST_LOOPBACK_IP - # (192.168.127.254) which is gvproxy's host-NAT entry, while - # gateway.containers.internal points at the gvproxy gateway itself. - if [ "${GATEWAY_IP}" = "${GVPROXY_GATEWAY_IP}" ]; then - printf '%s %s\n' "$GVPROXY_HOST_LOOPBACK_IP" "$host_aliases" >> "$hosts_tmp" - printf '%s %s\n' "$GVPROXY_GATEWAY_IP" "$gateway_aliases" >> "$hosts_tmp" - else - # TAP networking: gateway and host are both reachable at GATEWAY_IP. - printf '%s %s %s\n' "$GATEWAY_IP" "$host_aliases" "$gateway_aliases" >> "$hosts_tmp" - fi - if ! cat "$hosts_tmp" > "$hosts_path" 2>/dev/null; then - rm -f "$hosts_tmp" - ts "WARNING: could not update ${hosts_path}" - return 1 - fi - rm -f "$hosts_tmp" -} - -rewrite_openshell_endpoint_if_needed() { - local endpoint="${OPENSHELL_ENDPOINT:-}" - [ -n "$endpoint" ] || return 0 - - local parsed - if ! parsed="$(parse_endpoint "$endpoint")"; then - ts "WARNING: could not parse OPENSHELL_ENDPOINT=$endpoint" - return 0 - fi - - local scheme host port path - scheme="$(printf '%s\n' "$parsed" | sed -n '1p')" - host="$(printf '%s\n' "$parsed" | sed -n '2p')" - port="$(printf '%s\n' "$parsed" | sed -n '3p')" - path="$(printf '%s\n' "$parsed" | sed -n '4p')" - - if tcp_probe "$host" "$port"; then - return 0 - fi - - # Probe candidates in preference order. Hostnames first for informative - # log output, then a bare IP as a final safety net. In gvproxy mode the - # bare IP is the host-loopback (192.168.127.254). In TAP/GPU mode it's - # the TAP host gateway. - local fallback_ip="$GVPROXY_HOST_LOOPBACK_IP" - if [ "${GATEWAY_IP}" != "${GVPROXY_GATEWAY_IP}" ]; then - fallback_ip="$GATEWAY_IP" - fi - local candidates="host.openshell.internal host.containers.internal host.docker.internal" - if [ "$scheme" != "https" ]; then - candidates="${candidates} ${fallback_ip}" - fi - - for candidate in $candidates; do - if [ "$candidate" = "$host" ]; then - continue - fi - if tcp_probe "$candidate" "$port"; then - local authority="$candidate" - if ! { [ "$scheme" = "http" ] && [ "$port" = "80" ]; } \ - && ! { [ "$scheme" = "https" ] && [ "$port" = "443" ]; }; then - authority="${authority}:${port}" - fi - export OPENSHELL_ENDPOINT="${scheme}://${authority}${path}" - ts "rewrote OPENSHELL_ENDPOINT to ${OPENSHELL_ENDPOINT}" - return 0 - fi - done - - if [ "$scheme" = "https" ]; then - ts "WARNING: could not preflight HTTPS OpenShell endpoint ${host}:${port}; preserving hostname for TLS verification" - return 0 - fi - - ts "WARNING: could not reach OpenShell endpoint ${host}:${port}" -} - create_gpu_device_nodes_mknod() { # Mode 666 is intentional: single-tenant microVM with the VM itself as the # isolation boundary. The sandbox user is the only non-root user. @@ -752,104 +575,8 @@ run_post_overlay_setup() { configure_hostname ip link set lo up 2>/dev/null || true -# Networking: use TAP static config if VM_NET_IP is set (QEMU path), -# otherwise fall back to gvproxy DHCP on eth0 (libkrun path). -if [ -n "${VM_NET_IP}" ] && [ -n "${VM_NET_GW}" ]; then - ts "configuring TAP networking (static ${VM_NET_IP} gw ${VM_NET_GW})" - GATEWAY_IP="${VM_NET_GW}" - - TAP_NIC="" - NIC_WAIT=0 - while [ -z "$TAP_NIC" ] && [ "$NIC_WAIT" -lt 10 ]; do - for candidate in eth0 ens3 enp0s2; do - if ip link show "$candidate" >/dev/null 2>&1 && [ "$candidate" != "lo" ]; then - TAP_NIC="$candidate" - break - fi - done - if [ -z "$TAP_NIC" ]; then - for sys_nic in /sys/class/net/*; do - [ -e "$sys_nic" ] || continue - candidate="${sys_nic##*/}" - if ip link show "$candidate" >/dev/null 2>&1 && [ "$candidate" != "lo" ]; then - TAP_NIC="$candidate" - break - fi - done - fi - if [ -z "$TAP_NIC" ]; then - sleep 1 - NIC_WAIT=$((NIC_WAIT + 1)) - fi - done - - if [ -n "$TAP_NIC" ]; then - ts "using NIC ${TAP_NIC} for TAP networking" - ip link set "$TAP_NIC" up 2>/dev/null || true - ip addr add "${VM_NET_IP}/30" dev "$TAP_NIC" 2>/dev/null || true - ip route add default via "${VM_NET_GW}" 2>/dev/null || true - else - ts "WARNING: no network interface found for TAP networking" - fi - - if [ -n "${VM_NET_DNS}" ]; then - echo "nameserver ${VM_NET_DNS}" > "$(root_path /etc/resolv.conf)" - elif [ ! -s "$(root_path /etc/resolv.conf)" ]; then - echo "nameserver 8.8.8.8" > "$(root_path /etc/resolv.conf)" - echo "nameserver 8.8.4.4" >> "$(root_path /etc/resolv.conf)" - fi - - ensure_host_gateway_aliases -elif ip link show eth0 >/dev/null 2>&1; then - ts "detected eth0 (gvproxy networking)" - ip link set eth0 up 2>/dev/null || true - - if command -v udhcpc >/dev/null 2>&1; then - UDHCPC_SCRIPT="$(root_path /run/openshell-udhcpc.script)" - mkdir -p "$(dirname "$UDHCPC_SCRIPT")" - cat > "$UDHCPC_SCRIPT" <<'DHCP_SCRIPT' -#!/bin/sh -case "$1" in - bound|renew) - ip addr flush dev "$interface" - ip addr add "$ip/$mask" dev "$interface" - if [ -n "$router" ]; then - ip route add default via "$router" dev "$interface" - fi - if [ -n "$dns" ]; then - resolv_conf="${OPENSHELL_RESOLV_CONF:-/etc/resolv.conf}" - mkdir -p "$(dirname "$resolv_conf")" 2>/dev/null || true - : > "$resolv_conf" 2>/dev/null || true - for d in $dns; do - echo "nameserver $d" >> "$resolv_conf" 2>/dev/null || true - done - fi - ;; -esac -DHCP_SCRIPT - chmod +x "$UDHCPC_SCRIPT" - - if ! OPENSHELL_RESOLV_CONF="$(root_path /etc/resolv.conf)" \ - udhcpc -i eth0 -f -q -n -T 1 -t 3 -A 1 -s "$UDHCPC_SCRIPT" 2>&1; then - ts "WARNING: DHCP failed, falling back to static config" - ip addr add 192.168.127.2/24 dev eth0 2>/dev/null || true - ip route add default via "$GVPROXY_GATEWAY_IP" 2>/dev/null || true - fi - else - ts "no DHCP client, using static config" - ip addr add 192.168.127.2/24 dev eth0 2>/dev/null || true - ip route add default via "$GVPROXY_GATEWAY_IP" 2>/dev/null || true - fi - - if [ ! -s "$(root_path /etc/resolv.conf)" ]; then - echo "nameserver 8.8.8.8" > "$(root_path /etc/resolv.conf)" - echo "nameserver 8.8.4.4" >> "$(root_path /etc/resolv.conf)" - fi - - ensure_host_gateway_aliases -else - ts "WARNING: no network interface found; supervisor will start without guest egress" -fi +# The boundary transport mediates network and DNS requests. Only loopback is +# configured in the guest; no public resolver or guest NIC is needed. export HOME=/sandbox export USER=sandbox @@ -876,33 +603,16 @@ fi run_openshell_init_dropins -rewrite_openshell_endpoint_if_needed - -# Log supervisor connectivity state for debugging stuck-in-Provisioning issues -if [ -n "${OPENSHELL_ENDPOINT:-}" ]; then - _ep_parsed="$(parse_endpoint "$OPENSHELL_ENDPOINT" 2>/dev/null || true)" - if [ -n "$_ep_parsed" ]; then - _ep_host="$(printf '%s\n' "$_ep_parsed" | sed -n '2p')" - _ep_port="$(printf '%s\n' "$_ep_parsed" | sed -n '3p')" - if tcp_probe "$_ep_host" "$_ep_port"; then - ts "gateway reachable at ${_ep_host}:${_ep_port}" - else - ts "WARNING: gateway NOT reachable at ${_ep_host}:${_ep_port} — supervisor may fail to connect" - fi - fi - ts "OPENSHELL_ENDPOINT=${OPENSHELL_ENDPOINT}" -fi if [ -n "${OPENSHELL_SANDBOX_ID:-}" ]; then ts "OPENSHELL_SANDBOX_ID=${OPENSHELL_SANDBOX_ID}" fi -read_supervisor_extra_args - -ts "starting openshell-sandbox supervisor" +ts "starting OpenShell VM process leaf" if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then exec_supervisor_in_newroot fi -exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox "${SUPERVISOR_EXTRA_ARGS[@]+"${SUPERVISOR_EXTRA_ARGS[@]}"}" +exec /opt/openshell/bin/openshell-sandbox \ + --mode=boundary --boundary-config /etc/openshell/boundary.json } if [ "${1:-}" != "--post-overlay" ]; then diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 46133f38fe..fadf337d87 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1,9 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use crate::gpu::{ - GpuInventory, SubnetAllocator, allocate_vsock_cid, mac_from_sandbox_id, tap_device_name, -}; +#![allow(unsafe_code)] + +use crate::gpu::{GpuInventory, SubnetAllocator, allocate_vsock_cid}; + use crate::lifecycle::{ BackendFeature, GuestInitDropin, LaunchAbortReason, LaunchPlan, LifecycleExtensionRegistry, RestoreContext, extension_state_dir, @@ -11,8 +12,10 @@ use crate::lifecycle::{ use crate::rootfs::{ clone_or_copy_sparse_file, create_ext4_image_from_dir_with_size, create_rootfs_image_from_dir, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path, - set_rootfs_image_file_mode, write_rootfs_image_file, + sandbox_guest_runtime_identity, set_rootfs_image_file_mode, write_rootfs_image_file, }; +#[cfg(target_os = "linux")] +use crate::rootfs::{extract_host_supervisor, validate_host_supervisor}; use crate::runtime::VmBackend; use bollard::Docker; use bollard::errors::Error as BollardError; @@ -54,13 +57,19 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; +use openshell_isolation_interface::boundary_protocol::{ + BoundaryConfig, BoundaryTopology, BoundaryTransport, +}; use openshell_vfio::SysfsRoot; use opentelemetry::trace::TraceContextExt as _; use prost::Message; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; use std::fs; use std::future::Future; + +use crate::isolation::VmBoundarySpec; use std::io::Read; use std::net::{IpAddr, Ipv4Addr}; #[cfg(unix)] @@ -125,39 +134,38 @@ impl VmSandboxDriverConfig { } } -/// gvproxy host-loopback IP — gvproxy's TCP/UDP/ICMP forwarder NAT-rewrites -/// this destination to the host's `127.0.0.1` and dials out from the host -/// process. This is the only address that transparently reaches host-bound -/// services without explicit `expose` rules. -/// -/// See gvisor-tap-vsock `cmd/gvproxy/config.go` (default NAT entry -/// `HostIP -> 127.0.0.1`) and `pkg/services/forwarder/tcp.go` (NAT lookup -/// before `net.Dial`). -/// -/// Code paths route via `GVPROXY_HOST_LOOPBACK_ALIAS` (DNS / /etc/hosts) -/// instead so logs stay readable; this constant is kept for documentation -/// and parity with the guest init script. -#[allow(dead_code)] // Documentation/parity anchor; all routing goes via the alias. +/// Legacy gvproxy host-loopback IP accepted in configurations created before +/// the control process moved from the VM guest to the host. const GVPROXY_HOST_LOOPBACK_IP: &str = "192.168.127.254"; const OPENSHELL_HOST_GATEWAY_ALIAS: &str = "host.openshell.internal"; -/// Hostname gvproxy resolves (via its embedded DNS) to the host-loopback IP. -/// -/// We rewrite loopback URLs to this hostname rather than the bare IP because: -/// * the guest init script seeds /etc/hosts with the same mapping, so it -/// resolves even when gvproxy's DNS is not in resolv.conf; -/// * keeping a recognisable hostname makes log messages clearer than a bare -/// 192.168.127.254 reference; -/// * package-managed gateway certificates include this SAN for guest mTLS. -/// -/// Both names ultimately route through the gvproxy NAT path on -/// `GVPROXY_HOST_LOOPBACK_IP` — they do **not** go through the gateway IP. -const GVPROXY_HOST_LOOPBACK_ALIAS: &str = OPENSHELL_HOST_GATEWAY_ALIAS; +const LEGACY_HOST_GATEWAY_ALIASES: &[&str] = &[ + OPENSHELL_HOST_GATEWAY_ALIAS, + "host.containers.internal", + "host.docker.internal", + GVPROXY_HOST_LOOPBACK_IP, +]; +#[allow(dead_code)] const GUEST_SSH_SOCKET_PATH: &str = openshell_core::container_paths::SSH_SOCKET_PATH; +#[allow(dead_code)] const GUEST_TLS_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_CA_PATH; +#[allow(dead_code)] const GUEST_TLS_CERT_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_CERT_PATH; +#[allow(dead_code)] const GUEST_TLS_KEY_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_KEY_PATH; +#[allow(dead_code)] const GUEST_SANDBOX_TOKEN_PATH: &str = openshell_core::container_paths::VM_GUEST_SANDBOX_TOKEN_PATH; const GUEST_INIT_DROPIN_DIR: &str = openshell_core::container_paths::VM_GUEST_INIT_DROPIN_DIR; +const GUEST_BOUNDARY_CONFIG_PATH: &str = "/etc/openshell/boundary.json"; +const HOST_SANDBOX_TOKEN_FILE: &str = "sandbox.jwt"; +const HOST_TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload"; +/// The backend this driver's deployment admits, delivered to the supervisor on +/// a channel separate from the topology descriptor so descriptor verification +/// is not self-referential. +const DRIVER_ADMITTED_BACKEND: &str = "vm"; +#[cfg(target_os = "linux")] +const HOST_SUPERVISOR_BINARY: &str = "host-runtime/openshell-sandbox"; +const VM_CONTROL_SOCKET: &str = "control.sock"; +const VM_CONTROL_PORT: u32 = 5500; /// Guest path of the driver-authored manifest enumerating which /// `init.d` drop-ins the guest init script is allowed to execute. /// @@ -167,10 +175,12 @@ const GUEST_INIT_DROPIN_DIR: &str = openshell_core::container_paths::VM_GUEST_IN /// upperdir on every launch, so the image cannot forge or shadow it. const GUEST_INIT_DROPIN_MANIFEST: &str = openshell_core::container_paths::VM_GUEST_INIT_DROPIN_MANIFEST; -/// Guest path of the root-only corporate proxy credential staged by the driver. +/// Legacy guest path retained for migration tests of pre-boundary runtimes. +#[allow(dead_code)] const GUEST_UPSTREAM_PROXY_AUTH_PATH: &str = openshell_core::container_paths::VM_GUEST_UPSTREAM_PROXY_AUTH_PATH; -/// Guest path of the corporate proxy CA bundle staged by the driver. +/// Legacy guest path retained for migration tests of pre-boundary runtimes. +#[allow(dead_code)] const GUEST_PROXY_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_PROXY_CA_PATH; /// Guest path of the driver-authored supervisor argument list. /// @@ -178,6 +188,7 @@ const GUEST_PROXY_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_PROX /// command line: written into the overlay upperdir on every launch (empty /// when there is nothing to pass) so the guest appends exactly the arguments /// the driver chose and a sandbox image cannot forge or shadow them. +#[allow(dead_code)] const GUEST_SUPERVISOR_ARGS_PATH: &str = openshell_core::container_paths::VM_GUEST_SUPERVISOR_ARGS_PATH; const IMAGE_CACHE_ROOT_DIR: &str = "images"; @@ -194,7 +205,7 @@ const GUEST_IMAGE_CONFIG_DIR: &str = "openshell-image"; const GUEST_IMAGE_OCI_LAYOUT_DIR: &str = "oci"; const GUEST_IMAGE_OCI_REF: &str = "openshell"; const IMAGE_EXPORT_ROOTFS_ARCHIVE: &str = "source-rootfs.tar"; -const BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-bootstrap-rootfs-ext4-v3"; +const BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-bootstrap-rootfs-ext4-v4"; const PREPARED_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-prepared-rootfs-ext4-umoci-v3"; const IMAGE_IDENTITY_FILE: &str = "image-identity"; const IMAGE_REFERENCE_FILE: &str = "image-reference"; @@ -465,7 +476,7 @@ impl VmDriverConfig { if provided.iter().all(Option::is_none) { return if self.requires_tls_materials() { Err( - "https:// openshell endpoint requires OPENSHELL_VM_TLS_CA, OPENSHELL_VM_TLS_CERT, and OPENSHELL_VM_TLS_KEY so sandbox VMs can authenticate to the gateway" + "https:// openshell endpoint requires OPENSHELL_VM_TLS_CA, OPENSHELL_VM_TLS_CERT, and OPENSHELL_VM_TLS_KEY so the host supervisor can authenticate to the gateway" .to_string(), ) } else { @@ -524,9 +535,27 @@ fn validate_openshell_endpoint(endpoint: &str) -> Result<(), String> { Ok(()) } +fn host_control_openshell_endpoint(endpoint: &str) -> Result { + let mut url = Url::parse(endpoint) + .map_err(|err| format!("invalid openshell endpoint '{endpoint}': {err}"))?; + if url + .host_str() + .is_some_and(|host| LEGACY_HOST_GATEWAY_ALIASES.contains(&host)) + { + // These aliases historically addressed host loopback from inside the + // guest. Control mode now runs on the host, so dial loopback directly + // while preserving compatibility with existing VM configurations. + url.set_host(Some("127.0.0.1")).map_err(|error| { + format!("failed to rewrite openshell endpoint '{endpoint}': {error}") + })?; + } + Ok(url.into()) +} + #[derive(Debug)] struct VmProcess { child: Child, + supervisor: Child, deleting: bool, } @@ -585,7 +614,7 @@ impl VmDriver { } pub async fn new_with_extensions( - config: VmDriverConfig, + mut config: VmDriverConfig, lifecycle_extensions: LifecycleExtensionRegistry, ) -> Result { lifecycle_extensions @@ -598,6 +627,7 @@ impl VmDriver { } validate_openshell_endpoint(&config.openshell_endpoint)?; let _ = config.tls_paths()?; + config.state_dir = absolute_state_dir(&config.state_dir)?; #[cfg(target_os = "linux")] if config.gpu_enabled { @@ -663,6 +693,192 @@ impl VmDriver { Ok(driver) } + #[cfg_attr( + not(target_os = "linux"), + allow( + clippy::unused_async, + reason = "the shared call path awaits Linux extraction only" + ) + )] + async fn host_supervisor_binary(&self) -> Result { + if let Some(configured) = std::env::var_os("OPENSHELL_VM_SUPERVISOR_BIN") { + let configured = PathBuf::from(configured); + if configured.is_file() { + return Ok(configured); + } + return Err(Status::failed_precondition(format!( + "configured host supervisor does not exist: {}", + configured.display() + ))); + } + + if let Some(parent) = self.launcher_bin.parent() { + let sibling = parent.join("openshell-sandbox"); + if sibling.is_file() { + return Ok(sibling); + } + } + + #[cfg(not(target_os = "linux"))] + { + Err(Status::failed_precondition( + "the native host supervisor is missing; install openshell-sandbox beside openshell-driver-vm or set OPENSHELL_VM_SUPERVISOR_BIN", + )) + } + + #[cfg(target_os = "linux")] + { + let destination = self.config.state_dir.join(HOST_SUPERVISOR_BINARY); + if validate_host_supervisor(&destination).is_ok() { + return Ok(destination); + } + let _cache_guard = self.image_cache_lock.lock().await; + if validate_host_supervisor(&destination).is_ok() { + return Ok(destination); + } + let destination_for_extract = destination.clone(); + tokio::task::spawn_blocking(move || extract_host_supervisor(&destination_for_extract)) + .await + .map_err(|error| { + Status::internal(format!("host supervisor extraction panicked: {error}")) + })? + .map_err(Status::failed_precondition)?; + validate_host_supervisor(&destination).map_err(Status::failed_precondition)?; + Ok(destination) + } + } + + async fn spawn_host_supervisor( + &self, + sandbox: &Sandbox, + state_dir: &Path, + tls_paths: Option<&VmDriverTlsPaths>, + topology: &BoundaryTopology, + ) -> Result { + let supervisor_binary = self.host_supervisor_binary().await?; + let openshell_endpoint = host_control_openshell_endpoint(&self.config.openshell_endpoint) + .map_err(Status::failed_precondition)?; + let token = sandbox + .spec + .as_ref() + .map(|spec| spec.sandbox_token.as_str()) + .filter(|token| !token.is_empty()) + .ok_or_else(|| Status::failed_precondition("VM sandbox gateway token is required"))?; + let token_path = state_dir.join(HOST_SANDBOX_TOKEN_FILE); + tokio::fs::write(&token_path, format!("{token}\n")) + .await + .map_err(|error| Status::internal(format!("write host sandbox token: {error}")))?; + #[cfg(unix)] + tokio::fs::set_permissions(&token_path, fs::Permissions::from_mode(0o600)) + .await + .map_err(|error| Status::internal(format!("restrict host sandbox token: {error}")))?; + + let descriptor = topology + .descriptor(DRIVER_ADMITTED_BACKEND) + .map_err(|error| Status::internal(error.to_string()))?; + // The payload carries the boundary bootstrap token, so it must not + // appear in the world-readable process cmdline; deliver it through a + // driver-owned 0600 file like the gateway token. + let payload_path = state_dir.join(HOST_TOPOLOGY_PAYLOAD_FILE); + tokio::fs::write(&payload_path, &descriptor.payload) + .await + .map_err(|error| Status::internal(format!("write host topology payload: {error}")))?; + #[cfg(unix)] + tokio::fs::set_permissions(&payload_path, fs::Permissions::from_mode(0o600)) + .await + .map_err(|error| { + Status::internal(format!("restrict host topology payload: {error}")) + })?; + let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( + sandbox.spec.as_ref(), + ) + .map_err(|error| Status::internal(format!("encode main process spec: {error}")))?; + let sandbox_user_id = self.config.resolve_sandbox_uid(); + let primary_group_id = self.config.resolve_sandbox_gid(sandbox_user_id); + let mut command = Command::new(&supervisor_binary); + isolate_host_control_environment(&mut command); + command + .kill_on_drop(true) + .stdin(Stdio::null()) + .stdout(Stdio::from( + fs::File::create(state_dir.join("supervisor.log")) + .map_err(|error| Status::internal(format!("create supervisor log: {error}")))?, + )) + .stderr(Stdio::from( + fs::File::create(state_dir.join("supervisor.err.log")).map_err(|error| { + Status::internal(format!("create supervisor error log: {error}")) + })?, + )) + .arg("--mode=control") + .arg(format!( + "--topology-backend-name={}", + descriptor.backend_name + )) + .arg(format!("--topology-version={}", descriptor.version)) + .arg("--topology-payload-file") + .arg(&payload_path) + .arg("--workdir") + .arg("/sandbox") + .args(upstream_proxy_cli_args(&self.config)) + .env( + openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND, + DRIVER_ADMITTED_BACKEND, + ) + .env( + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, + main_process_spec, + ) + .env(openshell_core::sandbox_env::ENDPOINT, openshell_endpoint) + .env(openshell_core::sandbox_env::SANDBOX_ID, &sandbox.id) + .env(openshell_core::sandbox_env::SANDBOX, &sandbox.name) + .env(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, &token_path) + .env( + openshell_core::sandbox_env::SSH_SOCKET_PATH, + state_dir.join("ssh.sock"), + ) + .env( + openshell_core::sandbox_env::PROXY_TLS_DIR, + state_dir.join("proxy-tls"), + ) + .env( + openshell_core::sandbox_env::SANDBOX_UID, + sandbox_user_id.to_string(), + ) + .env( + openshell_core::sandbox_env::SANDBOX_GID, + primary_group_id.to_string(), + ) + .env(openshell_core::sandbox_env::OCI_IMAGE_USER, "") + .env( + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &self.config.log_level), + ) + .env( + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value(), + ); + configure_main_exit_marker(&mut command, state_dir); + if let Some(tls) = tls_paths { + command + .env(openshell_core::sandbox_env::TLS_CA, &tls.ca) + .env(openshell_core::sandbox_env::TLS_CERT, &tls.cert) + .env(openshell_core::sandbox_env::TLS_KEY, &tls.key); + } + #[cfg(target_os = "linux")] + unsafe { + command.pre_exec(|| { + nix::sys::prctl::set_pdeathsig(Signal::SIGKILL) + .map_err(|error| std::io::Error::other(error.to_string())) + }); + } + command.spawn().map_err(|error| { + Status::internal(format!( + "start host supervisor '{}': {error}", + supervisor_binary.display() + )) + }) + } + #[must_use] pub fn capabilities(&self) -> GetCapabilitiesResponse { GetCapabilitiesResponse { @@ -893,6 +1109,8 @@ impl VmDriver { let root_disk = image_plan.root_disk; let image_disk = image_plan.image_disk; let overlay_disk = disk_paths.overlay_disk; + let bootstrap_token = random_boundary_token(); + let boundary_generation = random_boundary_token(); self.publish_platform_event( sandbox.id.clone(), @@ -904,16 +1122,7 @@ impl VmDriver { ), ); if let Err(err) = self - .prepare_runtime_overlay( - &overlay_disk, - tls_paths.as_ref(), - sandbox - .spec - .as_ref() - .map(|spec| spec.sandbox_token.as_str()) - .filter(|token| !token.is_empty()), - overlay_preparation, - ) + .prepare_runtime_overlay(&overlay_disk, overlay_preparation) .await { return Err(Status::internal(format!( @@ -951,19 +1160,6 @@ impl VmDriver { } }; - // `build_vm_launch_plan` already allocated the QEMU subnet, so record - // it as allocated now — before the cancellable `configure_launch` / - // `before_launch` hooks run. If a delete aborts provisioning while - // one of those hooks is awaiting, the aborted future never runs its - // own release path, and the delete cleanup is gated on this flag; if - // the flag were still unset the subnet would leak. - if plan.backend == VmBackend::Qemu - && let Err(err) = self.mark_qemu_network_allocated(&sandbox.id).await - { - self.release_gpu_and_subnet(&sandbox.id); - return Err(err); - } - if let Err(err) = self .lifecycle_extensions .configure_launch(&sandbox, &state_dir, &mut plan) @@ -1054,25 +1250,40 @@ impl VmDriver { return Err(err); } - // Staged on every launch, including a restart onto a preserved - // overlay, so the driver's copy always shadows the image layer. - if let Err(err) = inject_guest_upstream_proxy(&overlay_disk, &self.config).await { - self.lifecycle_extensions - .after_launch_failed(&sandbox, &state_dir, LaunchAbortReason::GuestPrepareFailed) - .await; - self.release_gpu_and_subnet(&sandbox.id); - return Err(err); - } - - let endpoint_override = if plan.backend == VmBackend::Qemu { - plan.host_ip.as_deref().map(|host_ip| { - guest_visible_openshell_endpoint_for_tap(&self.config.openshell_endpoint, host_ip) - }) + let console_output = state_dir.join("rootfs-console.log"); + let control_socket = state_dir.join(VM_CONTROL_SOCKET); + let transport = if plan.backend == VmBackend::Qemu { + BoundaryTransport::Vsock { + guest_cid: plan.vsock_cid.ok_or_else(|| { + Status::internal("QEMU launch plan is missing a guest vsock CID") + })?, + control_port: VM_CONTROL_PORT, + } } else { - None + BoundaryTransport::Unix { + socket_path: control_socket.clone(), + } }; - - let console_output = state_dir.join("rootfs-console.log"); + let sandbox_user_id = self.config.resolve_sandbox_uid(); + let provisioning = VmBoundarySpec { + boundary_id: sandbox.id.clone(), + bootstrap_token, + generation: boundary_generation, + image_identity, + transport, + control_port: VM_CONTROL_PORT, + agent_uid: sandbox_user_id, + agent_gid: self.config.resolve_sandbox_gid(sandbox_user_id), + trusted_runtime_root: PathBuf::from( + "/.openshell-bootstrap/opt/openshell/bin/openshell-runtime", + ), + child_env: merged_environment(&sandbox), + } + .provision(); + inject_guest_boundary_config(&overlay_disk, &provisioning.boundary_config).map_err( + |error| Status::internal(format!("inject VM boundary configuration: {error}")), + )?; + let topology = provisioning.topology; let mut command = Command::new(&self.launcher_bin); command.kill_on_drop(true); command.stdin(Stdio::null()); @@ -1116,6 +1327,13 @@ impl VmDriver { if let Some(port) = plan.gateway_port { command.arg("--vm-gateway-port").arg(port.to_string()); } + } else { + let _ = tokio::fs::remove_file(&control_socket).await; + command + .arg("--vm-vsock-control-port") + .arg(VM_CONTROL_PORT.to_string()) + .arg("--vm-vsock-control-socket") + .arg(&control_socket); } self.ensure_provisioning_active(&sandbox.id).await?; @@ -1124,7 +1342,7 @@ impl VmDriver { .arg("--vm-krun-log-level") .arg(self.config.krun_log_level.to_string()); - for env in build_guest_environment(&sandbox, &self.config, endpoint_override.as_deref()) { + for env in build_guest_environment(&sandbox, &self.config) { command.arg("--vm-env").arg(env); } for env in &plan.env { @@ -1137,7 +1355,7 @@ impl VmDriver { console_output = %console_output.display(), "vm driver: spawning VM launcher" ); - let child = match spawn_vm_launcher(&mut command, &sandbox.id, &plan.backend) { + let mut child = match command.spawn() { Ok(child) => child, Err(err) => { warn!( @@ -1164,8 +1382,27 @@ impl VmDriver { launcher_pid = child.id().unwrap_or(0), "vm driver: launcher spawned" ); + let supervisor = match self + .spawn_host_supervisor(&sandbox, &state_dir, tls_paths.as_ref(), &topology) + .await + { + Ok(supervisor) => supervisor, + Err(error) => { + let _ = terminate_vm_process(&mut child).await; + self.lifecycle_extensions + .after_launch_failed( + &sandbox, + &state_dir, + LaunchAbortReason::LauncherSpawnFailed, + ) + .await; + self.release_gpu_and_subnet(&sandbox.id); + return Err(error); + } + }; let process = Arc::new(Mutex::new(VmProcess { child, + supervisor, deleting: false, })); @@ -1177,7 +1414,7 @@ impl VmDriver { Some(record) if !record.deleting => { record.process = Some(process.clone()); record.gpu_bdf.clone_from(&gpu_bdf); - record.qemu_network_allocated = plan.backend == VmBackend::Qemu; + record.qemu_network_allocated = false; snapshot_to_publish = Some(record.snapshot.clone()); } _ => { @@ -1190,6 +1427,9 @@ impl VmDriver { { let mut process = process.lock().await; process.deleting = true; + terminate_vm_process(&mut process.supervisor) + .await + .map_err(|err| Status::internal(format!("failed to stop supervisor: {err}")))?; terminate_vm_process(&mut process.child) .await .map_err(|err| Status::internal(format!("failed to stop vm: {err}")))?; @@ -1275,6 +1515,9 @@ impl VmDriver { if let Some(process) = process { let mut process = process.lock().await; process.deleting = true; + terminate_vm_process(&mut process.supervisor) + .await + .map_err(|err| Status::internal(format!("failed to stop supervisor: {err}")))?; terminate_vm_process(&mut process.child) .await .map_err(|err| Status::internal(format!("failed to stop vm: {err}")))?; @@ -1419,6 +1662,9 @@ impl VmDriver { if let Some(process) = process { let mut process = process.lock().await; process.deleting = true; + terminate_vm_process(&mut process.supervisor) + .await + .map_err(|err| Status::internal(format!("failed to stop supervisor: {err}")))?; terminate_vm_process(&mut process.child) .await .map_err(|err| Status::internal(format!("failed to stop vm: {err}")))?; @@ -1834,10 +2080,12 @@ impl VmDriver { Ok(()) } - #[allow(clippy::result_large_err)] + // Keep the fallible shape used by launch-plan resolution: driver-local + // backends may add allocation failures here without changing callers. + #[allow(clippy::result_large_err, clippy::unnecessary_wraps)] fn configure_qemu_launch_plan( &self, - sandbox_id: &str, + _sandbox_id: &str, is_gpu: bool, gpu_bdf: Option, plan: &mut LaunchPlan, @@ -1850,44 +2098,15 @@ impl VmDriver { if plan.gpu_bdf.is_none() { plan.gpu_bdf = gpu_bdf; } - if !has_complete_qemu_network(plan) { - let subnet = self - .subnet_allocator - .lock() - .map_err(|e| Status::internal(format!("subnet allocator lock poisoned: {e}")))? - .allocate(sandbox_id) - .map_err(Status::failed_precondition)?; - let mac = mac_from_sandbox_id(sandbox_id); - plan.tap_device = Some(tap_device_name(sandbox_id)); - plan.guest_ip = Some(subnet.guest_ip.to_string()); - plan.host_ip = Some(subnet.host_ip.to_string()); - plan.vsock_cid = Some(allocate_vsock_cid()); - plan.guest_mac = Some(format!( - "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", - mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] - )); - plan.gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); - } - - // The corporate-proxy host-loopback recipe is a libkrun/gvproxy - // property and has no QEMU/TAP equivalent (see - // `proxy_url_targets_gateway_host`). Run it here, after the subnet - // allocation above has settled `plan.host_ip`, because the address to - // compare against is this sandbox's own TAP host address. Fail the - // create with the reason rather than boot a sandbox whose - // policy-approved CONNECTs all time out against an unreachable proxy. - if let Some(url) = self.config.https_proxy.as_deref() - && proxy_url_targets_gateway_host(url, plan.host_ip.as_deref()) - { - let tap_host = plan.host_ip.as_deref().unwrap_or("the TAP host address"); - return Err(Status::failed_precondition(format!( - "https_proxy '{url}' addresses the gateway host, which a QEMU/TAP sandbox \ - (GPU sandboxes) cannot reach: host.openshell.internal resolves to this \ - sandbox's TAP host address {tap_host} and the driver's nftables rules allow \ - only the gateway port from the guest. Configure a proxy address routable \ - from the guest's masqueraded egress, or run this sandbox without a GPU" - ))); + if plan.vsock_cid.is_some() { + return Ok(()); } + plan.vsock_cid = Some(allocate_vsock_cid()); + plan.tap_device = None; + plan.guest_ip = None; + plan.host_ip = None; + plan.guest_mac = None; + plan.gateway_port = None; Ok(()) } @@ -1966,21 +2185,12 @@ impl VmDriver { Ok(()) } - async fn mark_qemu_network_allocated(&self, sandbox_id: &str) -> Result<(), Status> { - let mut registry = self.registry.lock().await; - match registry.get_mut(sandbox_id) { - Some(record) if !record.deleting => { - record.qemu_network_allocated = true; - Ok(()) - } - _ => Err(Status::cancelled("sandbox provisioning cancelled")), - } - } - - #[allow(clippy::result_large_err)] + // Keep the fallible shape used by provisioning and lifecycle tests even + // though NIC/subnet allocation no longer introduces a failure today. + #[allow(clippy::result_large_err, clippy::unnecessary_wraps)] fn build_vm_launch_plan( &self, - sandbox_id: &str, + _sandbox_id: &str, needs_qemu: bool, is_gpu: bool, gpu_bdf: Option, @@ -2006,21 +2216,7 @@ impl VmDriver { }); } - let subnet = self - .subnet_allocator - .lock() - .map_err(|e| Status::internal(format!("subnet allocator lock poisoned: {e}")))? - .allocate(sandbox_id) - .map_err(Status::failed_precondition)?; let vsock_cid = allocate_vsock_cid(); - let mac = mac_from_sandbox_id(sandbox_id); - let mac_str = format!( - "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", - mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] - ); - let tap = tap_device_name(sandbox_id); - let gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); - let (vcpus, mem_mib) = if is_gpu { (self.config.gpu_vcpus, self.config.gpu_mem_mib) } else { @@ -2036,12 +2232,12 @@ impl VmDriver { kernel_profile: None, kernel_image: None, gpu_bdf, - tap_device: Some(tap), - guest_ip: Some(subnet.guest_ip.to_string()), - host_ip: Some(subnet.host_ip.to_string()), + tap_device: None, + guest_ip: None, + host_ip: None, vsock_cid: Some(vsock_cid), - guest_mac: Some(mac_str), - gateway_port, + guest_mac: None, + gateway_port: None, guest_init_dropins: Vec::new(), env: Vec::new(), }) @@ -2217,16 +2413,9 @@ impl VmDriver { async fn prepare_runtime_overlay( &self, overlay_disk: &Path, - tls_paths: Option<&VmDriverTlsPaths>, - sandbox_token: Option<&str>, preparation: OverlayPreparation, ) -> Result<(), String> { let span_status = openshell_otel::ErrorStatusGuard::current(); - let tls_materials = match tls_paths { - Some(paths) => Some(read_guest_tls_materials(paths).await?), - None => None, - }; - let sandbox_token = sandbox_token.map(str::to_string); let overlay_disk = overlay_disk.to_path_buf(); let overlay_size_bytes = self .config @@ -2254,8 +2443,6 @@ impl VmDriver { prepare_sandbox_overlay_image( &template_path, &overlay_disk, - tls_materials.as_ref(), - sandbox_token.as_deref(), preparation, overlay_size_bytes, ) @@ -3347,39 +3534,48 @@ impl VmDriver { process.clone() }; - let exit_status = { + let poll_result = { let mut process = process.lock().await; if process.deleting { return; } match process.child.try_wait() { - Ok(status) => status, - Err(err) => { - if let Some(snapshot) = self - .set_snapshot_condition( - &sandbox_id, - error_condition("ProcessPollFailed", &err.to_string()), - false, - ) - .await - { - self.publish_snapshot(snapshot); - } - self.publish_platform_event( - sandbox_id.clone(), - platform_event( - "vm", - "Warning", - "ProcessPollFailed", - format!("Failed to poll VM helper process: {err}"), - ), - ); - return; + Ok(Some(status)) => Ok(Some(("VM", status))), + Ok(None) => process + .supervisor + .try_wait() + .map(|status| status.map(|status| ("host supervisor", status))), + Err(error) => Err(error), + } + }; + + let exit_status = match poll_result { + Ok(status) => status, + Err(err) => { + if let Some(snapshot) = self + .set_snapshot_condition( + &sandbox_id, + error_condition("ProcessPollFailed", &err.to_string()), + false, + ) + .await + { + self.publish_snapshot(snapshot); } + self.publish_platform_event( + sandbox_id.clone(), + platform_event( + "vm", + "Warning", + "ProcessPollFailed", + format!("Failed to poll VM sandbox process: {err}"), + ), + ); + return; } }; - if let Some(status) = exit_status { + if let Some((component, status)) = exit_status { let state_dir = { let registry = self.registry.lock().await; registry @@ -3399,9 +3595,17 @@ impl VmDriver { "vm driver: failed to persist canonical-process exit tombstone" ); } + { + let mut process = process.lock().await; + if component == "VM" { + let _ = terminate_vm_process(&mut process.supervisor).await; + } else { + let _ = terminate_vm_process(&mut process.child).await; + } + } let message = status.code().map_or_else( - || "VM process exited".to_string(), - |code| format!("VM process exited with status {code}"), + || format!("{component} process exited"), + |code| format!("{component} process exited with status {code}"), ); if let Some(snapshot) = self .set_snapshot_condition( @@ -3504,6 +3708,12 @@ impl VmDriver { } } +fn configure_main_exit_marker(command: &mut Command, state_dir: &Path) { + command + .arg("--main-exit-marker") + .arg(state_dir.join(MAIN_PROCESS_EXITED_FILE)); +} + #[tonic::async_trait] impl ComputeDriver for VmDriver { async fn authenticate_sandbox( @@ -3699,8 +3909,8 @@ impl ComputeDriver for VmDriver { fn check_gpu_privileges() -> Result<(), String> { if !rustix::process::geteuid().is_root() { return Err( - "GPU support requires root privileges for VFIO bind/unbind and TAP networking. \ - Run with sudo or ensure CAP_SYS_ADMIN + CAP_NET_ADMIN capabilities are set." + "GPU support requires root privileges for VFIO bind/unbind. \ + Run with sudo or grant the host device-management capabilities required by VFIO." .to_string(), ); } @@ -4718,42 +4928,12 @@ fn merged_environment(sandbox: &Sandbox) -> HashMap { environment } -/// Rewrites loopback host references in a gateway URL to a hostname the guest -/// can reach via gvproxy. -/// -/// The driver receives the gateway endpoint from `--openshell-endpoint`, which -/// in local/dev/e2e setups is typically `http://127.0.0.1:`. That URL is -/// useless inside the guest because the guest's loopback interface is its own, -/// not the host's. Inside the guest we need a name that gvproxy will translate -/// into the host's loopback address. -/// -/// We rewrite to `host.openshell.internal`, which gvproxy's embedded DNS resolves -/// to the host-loopback IP `192.168.127.254`. gvproxy installs a default NAT entry -/// rewriting that destination to the host's `127.0.0.1` and dialing out from the -/// host process, so any port the host is listening on becomes reachable. The -/// gateway IP `192.168.127.1` does **not** do this — it only listens on gvproxy's -/// own service ports (DNS, DHCP, HTTP API). The guest init script also seeds the -/// hostname in `/etc/hosts` so resolution works even if gvproxy's DNS isn't in -/// resolv.conf (e.g. when DHCP fails). -/// -/// Non-loopback URLs are returned unchanged. -fn guest_visible_openshell_endpoint(endpoint: &str) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); - }; - - let should_rewrite = match url.host() { - Some(Host::Ipv4(ip)) => ip.is_loopback(), - Some(Host::Ipv6(ip)) => ip.is_loopback(), - Some(Host::Domain(host)) => host.eq_ignore_ascii_case("localhost"), - None => false, - }; - - if should_rewrite && url.set_host(Some(GVPROXY_HOST_LOOPBACK_ALIAS)).is_ok() { - return url.to_string(); +fn random_boundary_token() -> String { + let mut token = String::with_capacity(64); + for byte in rand::random::<[u8; 32]>() { + write!(&mut token, "{byte:02x}").expect("writing to String cannot fail"); } - - endpoint.to_string() + token } /// Whether a corporate proxy URL points at the gateway host itself, as seen @@ -4784,6 +4964,7 @@ fn guest_visible_openshell_endpoint(endpoint: &str) -> String { /// /// Used to reject an unreachable configuration up front on the QEMU path /// instead of letting every policy-approved CONNECT time out. +#[allow(dead_code)] fn proxy_url_targets_gateway_host(url: &str, tap_host_ip: Option<&str>) -> bool { let Ok(parsed) = Url::parse(url) else { // Unparseable URLs are rejected by shared validation before launch. @@ -4803,62 +4984,22 @@ fn proxy_url_targets_gateway_host(url: &str, tap_host_ip: Option<&str>) -> bool } } +#[allow(dead_code)] fn gateway_port_from_endpoint(endpoint: &str) -> Option { Url::parse(endpoint).ok().and_then(|url| url.port()) } -fn has_complete_qemu_network(plan: &LaunchPlan) -> bool { - plan.tap_device.is_some() - && plan.guest_ip.is_some() - && plan.host_ip.is_some() - && plan.vsock_cid.is_some() - && plan.guest_mac.is_some() -} - -fn guest_visible_openshell_endpoint_for_tap(endpoint: &str, host_ip: &str) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); - }; - if url.set_host(Some(host_ip)).is_ok() { - url.to_string() - } else { - endpoint.to_string() - } -} - -fn build_guest_environment( - sandbox: &Sandbox, - config: &VmDriverConfig, - endpoint_override: Option<&str>, -) -> Vec { - let openshell_endpoint = endpoint_override.map_or_else( - || guest_visible_openshell_endpoint(&config.openshell_endpoint), - String::from, - ); - // 1. User-supplied environment (lowest priority). - let user_env = merged_environment(sandbox); +fn build_guest_environment(sandbox: &Sandbox, config: &VmDriverConfig) -> Vec { + // The guest receives only driver-owned boot metadata. Gateway credentials, + // TLS material, and logical-supervisor configuration remain on the host; + // workload environment is carried in the authenticated BoundaryConfig. let mut environment: HashMap = HashMap::new(); - environment.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - environment.insert( - openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), - json, - ); - } - - // 2. Required driver vars (highest priority -- always overwrite). environment.insert("HOME".to_string(), "/root".to_string()); environment.insert( "PATH".to_string(), "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(), ); environment.insert("TERM".to_string(), "xterm".to_string()); - environment.insert( - openshell_core::sandbox_env::ENDPOINT.to_string(), - openshell_endpoint, - ); environment.insert( openshell_core::sandbox_env::SANDBOX_ID.to_string(), sandbox.id.clone(), @@ -4867,68 +5008,14 @@ fn build_guest_environment( openshell_core::sandbox_env::SANDBOX.to_string(), sandbox.name.clone(), ); - environment.insert( - openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), - GUEST_SSH_SOCKET_PATH.to_string(), - ); - // The libkrun guest environment path does not preserve spaces in values - // before guest startup. Use a whitespace-free base64url envelope so - // command arguments remain lossless. - let main_process = - openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec_base64url( - sandbox.spec.as_ref(), - ) - .expect("main process config serialization cannot fail"); - environment.insert( - openshell_core::sandbox_env::MAIN_PROCESS_SPEC.to_string(), - main_process, - ); environment.insert( openshell_core::sandbox_env::LOG_LEVEL.to_string(), openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), ); - if config.requires_tls_materials() { - environment.insert( - openshell_core::sandbox_env::TLS_CA.to_string(), - GUEST_TLS_CA_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_CERT.to_string(), - GUEST_TLS_CERT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_KEY.to_string(), - GUEST_TLS_KEY_PATH.to_string(), - ); - } environment.insert( openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), openshell_core::telemetry::enabled_env_value().to_string(), ); - // Runtime capabilities are driver-owned. The VM driver does not yet - // provide policy DNS and transparent TCP interception. - environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - String::new(), - ); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - // Prevent user-supplied environment from overriding the TLS server name - // the supervisor verifies — a sandbox user who can redirect the gateway - // hostname could otherwise present a certificate for a name they control - // and intercept the sandbox JWT. - environment.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - if sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.is_empty()) - { - environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), - GUEST_SANDBOX_TOKEN_PATH.to_string(), - ); - } - let mut pairs = environment.into_iter().collect::>(); pairs.sort_by(|left, right| left.0.cmp(&right.0)); pairs @@ -5136,8 +5223,9 @@ fn write_oci_layout_for_manifest( fn bootstrap_image_cache_identity(image_identity: &str) -> String { format!( - "{BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{image_identity}", - openshell_core::VERSION + "{BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:guest-{}:{image_identity}", + openshell_core::VERSION, + sandbox_guest_runtime_identity() ) } @@ -5256,26 +5344,6 @@ fn validate_restored_sandbox_state( Ok(()) } -#[derive(Debug, Clone)] -struct GuestTlsMaterials { - ca: Vec, - cert: Vec, - key: Vec, -} - -async fn read_guest_tls_materials(paths: &VmDriverTlsPaths) -> Result { - let ca = tokio::fs::read(&paths.ca) - .await - .map_err(|err| format!("read {}: {err}", paths.ca.display()))?; - let cert = tokio::fs::read(&paths.cert) - .await - .map_err(|err| format!("read {}: {err}", paths.cert.display()))?; - let key = tokio::fs::read(&paths.key) - .await - .map_err(|err| format!("read {}: {err}", paths.key.display()))?; - Ok(GuestTlsMaterials { ca, cert, key }) -} - async fn overlay_template_image_ready(path: &Path, size_bytes: u64) -> Result { match tokio::fs::metadata(path).await { Ok(metadata) => Ok(metadata.is_file() && metadata.len() == size_bytes), @@ -5360,36 +5428,19 @@ fn create_empty_sandbox_overlay_image(overlay_disk: &Path, size_bytes: u64) -> R fn create_sandbox_overlay_image_from_template( template_path: &Path, overlay_disk: &Path, - tls_materials: Option<&GuestTlsMaterials>, - sandbox_token: Option<&str>, ) -> Result<(), String> { - clone_or_copy_sparse_file(template_path, overlay_disk)?; - if let Some(tls) = tls_materials { - inject_guest_tls_materials(overlay_disk, tls)?; - } - if let Some(token) = sandbox_token { - inject_guest_sandbox_token(overlay_disk, token)?; - } - Ok(()) + clone_or_copy_sparse_file(template_path, overlay_disk) } fn prepare_sandbox_overlay_image( template_path: &Path, overlay_disk: &Path, - tls_materials: Option<&GuestTlsMaterials>, - sandbox_token: Option<&str>, preparation: OverlayPreparation, expected_size_bytes: u64, ) -> Result<(), String> { if preparation == OverlayPreparation::PreserveExisting { match fs::metadata(overlay_disk) { Ok(metadata) if metadata.is_file() && metadata.len() == expected_size_bytes => { - if let Some(tls) = tls_materials { - inject_guest_tls_materials(overlay_disk, tls)?; - } - if let Some(token) = sandbox_token { - inject_guest_sandbox_token(overlay_disk, token)?; - } return Ok(()); } Ok(metadata) if metadata.is_file() => { @@ -5416,37 +5467,19 @@ fn prepare_sandbox_overlay_image( } } - create_sandbox_overlay_image_from_template( - template_path, - overlay_disk, - tls_materials, - sandbox_token, - ) + create_sandbox_overlay_image_from_template(template_path, overlay_disk) } -fn inject_guest_tls_materials( +fn inject_guest_boundary_config( overlay_disk: &Path, - materials: &GuestTlsMaterials, + config: &BoundaryConfig, ) -> Result<(), String> { - write_rootfs_image_file( - overlay_disk, - &overlay_upper_path(GUEST_TLS_CA_PATH), - &materials.ca, - )?; - write_rootfs_image_file( - overlay_disk, - &overlay_upper_path(GUEST_TLS_CERT_PATH), - &materials.cert, - )?; - let key_path = overlay_upper_path(GUEST_TLS_KEY_PATH); - write_rootfs_image_file(overlay_disk, &key_path, &materials.key)?; - set_rootfs_image_file_mode(overlay_disk, &key_path, 0o600) -} - -fn inject_guest_sandbox_token(overlay_disk: &Path, token: &str) -> Result<(), String> { - let token_path = overlay_upper_path(GUEST_SANDBOX_TOKEN_PATH); - write_rootfs_image_file(overlay_disk, &token_path, format!("{token}\n").as_bytes())?; - set_rootfs_image_file_mode(overlay_disk, &token_path, 0o600) + let config = config + .encode() + .map_err(|error| format!("encode VM boundary configuration: {error}"))?; + let config_path = overlay_upper_path(GUEST_BOUNDARY_CONFIG_PATH); + write_rootfs_image_file(overlay_disk, &config_path, &config)?; + set_rootfs_image_file_mode(overlay_disk, &config_path, 0o600) } #[allow(clippy::result_large_err)] @@ -5496,12 +5529,12 @@ fn inject_guest_init_dropins( span_status.finish(Ok(())) } -/// Build the corporate upstream-proxy arguments passed to the guest supervisor. +/// Build the corporate upstream-proxy arguments passed to host control. /// /// This operator-owned egress boundary travels on the supervisor's argv, /// which sandbox spec/template environment and image `ENV` cannot influence. -/// Credentials are never on argv — only the root-only guest path is passed; -/// the supervisor reads the credential from that file. +/// Credentials are never on argv; the supervisor reads them from the +/// operator-owned host file. fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec { let mut args = Vec::new(); if let Some(url) = &config.https_proxy { @@ -5512,10 +5545,9 @@ fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec { args.push("--upstream-no-proxy".to_string()); args.push(list.clone()); } - if config.proxy_auth_file.is_some() { + if let Some(path) = &config.proxy_auth_file { args.push("--upstream-proxy-auth-file".to_string()); - // The guest path, never the gateway-host path the operator configured. - args.push(GUEST_UPSTREAM_PROXY_AUTH_PATH.to_string()); + args.push(path.clone()); } // Config validation guarantees the acknowledgement is `true` whenever an // auth file is configured against an http:// proxy; the supervisor @@ -5528,9 +5560,9 @@ fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec { if config.proxy_connect_by_hostname == Some(true) { args.push("--upstream-proxy-connect-by-hostname".to_string()); } - if config.proxy_ca_bundle.is_some() { + if let Some(path) = &config.proxy_ca_bundle { args.push("--upstream-proxy-ca-bundle".to_string()); - args.push(GUEST_PROXY_CA_PATH.to_string()); + args.push(path.clone()); } args } @@ -5541,6 +5573,7 @@ fn upstream_proxy_cli_args(config: &VmDriverConfig) -> Vec { /// without word splitting or globbing, so values containing spaces survive /// intact. An empty list renders an empty file, which the guest reads as "no /// extra arguments". +#[allow(dead_code)] fn render_guest_supervisor_args(args: &[String]) -> Vec { let mut body = args.join("\n"); if !body.is_empty() { @@ -5554,6 +5587,7 @@ fn render_guest_supervisor_args(args: &[String]) -> Vec { /// Every value here is operator-supplied config, so this is a guard against /// misconfiguration rather than an attack: a stray newline would otherwise /// split one value into two arguments in the guest. +#[allow(dead_code)] fn validate_guest_supervisor_args(args: &[String]) -> Result<(), String> { for arg in args { if arg.contains('\n') || arg.contains('\r') || arg.contains('\0') { @@ -5570,6 +5604,7 @@ fn validate_guest_supervisor_args(args: &[String]) -> Result<(), String> { /// Uses the validators shared with the supervisor, so a credential accepted /// here is never rejected inside the guest. The error never carries the file /// contents. +#[allow(dead_code)] async fn read_sandbox_proxy_credential(path: &str) -> Result { let path_owned = path.to_string(); let raw = tokio::task::spawn_blocking(move || { @@ -5592,6 +5627,7 @@ async fn read_sandbox_proxy_credential(path: &str) -> Result { /// Checked here rather than only in the guest so the operator gets an error /// attributable to `proxy_ca_bundle` instead of an opaque supervisor startup /// failure inside every sandbox. The error never carries the file contents. +#[allow(dead_code)] async fn read_sandbox_proxy_ca_bundle(path: &str) -> Result, Status> { let path_owned = path.to_string(); let pem = tokio::task::spawn_blocking(move || { @@ -5629,6 +5665,7 @@ async fn read_sandbox_proxy_ca_bundle(path: &str) -> Result, Status> { /// delivery the per-sandbox gateway JWT already uses. It is removed with the /// sandbox when the state directory is deleted. #[allow(clippy::result_large_err)] +#[allow(dead_code)] async fn inject_guest_upstream_proxy( overlay_disk: &Path, config: &VmDriverConfig, @@ -5869,47 +5906,6 @@ fn dir_size_bytes(path: &Path) -> Result { Ok(total) } -#[cfg(test)] -fn stage_guest_tls_materials( - staging_dir: &Path, - materials: &GuestTlsMaterials, -) -> Result<(), String> { - let tls_dir = staging_dir - .join("upper") - .join(GUEST_TLS_CA_PATH.trim_start_matches('/')) - .parent() - .ok_or_else(|| "guest TLS CA path has no parent".to_string())? - .to_path_buf(); - fs::create_dir_all(&tls_dir) - .map_err(|err| format!("create guest TLS dir {}: {err}", tls_dir.display()))?; - - let ca_path = staging_dir - .join("upper") - .join(GUEST_TLS_CA_PATH.trim_start_matches('/')); - let cert_path = staging_dir - .join("upper") - .join(GUEST_TLS_CERT_PATH.trim_start_matches('/')); - let key_path = staging_dir - .join("upper") - .join(GUEST_TLS_KEY_PATH.trim_start_matches('/')); - fs::write(&ca_path, &materials.ca) - .map_err(|err| format!("write guest TLS CA {}: {err}", ca_path.display()))?; - fs::write(&cert_path, &materials.cert) - .map_err(|err| format!("write guest TLS cert {}: {err}", cert_path.display()))?; - fs::write(&key_path, &materials.key) - .map_err(|err| format!("write guest TLS key {}: {err}", key_path.display()))?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - - fs::set_permissions(&key_path, fs::Permissions::from_mode(0o600)) - .map_err(|err| format!("chmod guest TLS key {}: {err}", key_path.display()))?; - } - - Ok(()) -} - fn overlay_staging_dir(overlay_disk: &Path) -> PathBuf { let parent = overlay_disk.parent().unwrap_or_else(|| Path::new(".")); parent.join(format!( @@ -5939,6 +5935,19 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> { } } +fn absolute_state_dir(state_dir: &Path) -> Result { + if state_dir.is_absolute() { + return Ok(state_dir.to_path_buf()); + } + std::env::current_dir() + .map(|working_dir| working_dir.join(state_dir)) + .map_err(|err| format!("failed to resolve VM driver state directory: {err}")) +} + +fn isolate_host_control_environment(command: &mut Command) { + command.env_clear(); +} + #[tracing::instrument( name = "vm.launch", skip(command), @@ -5949,6 +5958,7 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> { vm.backend = ?backend, ) )] +#[allow(dead_code)] fn spawn_vm_launcher( command: &mut Command, sandbox_id: &str, @@ -6653,7 +6663,7 @@ mod tests { let parent = tracing::info_span!("vm.provision"); let result = driver - .prepare_runtime_overlay(Path::new("/unused"), None, None, OverlayPreparation::Fresh) + .prepare_runtime_overlay(Path::new("/unused"), OverlayPreparation::Fresh) .instrument(parent) .await; assert!(result.is_err(), "overflow should stop before disk I/O"); @@ -7385,8 +7395,6 @@ mod tests { prepare_sandbox_overlay_image( &template, &overlay, - None, - None, OverlayPreparation::PreserveExisting, "saved-overlay".len() as u64, ) @@ -7408,8 +7416,6 @@ mod tests { prepare_sandbox_overlay_image( &template, &overlay, - None, - None, OverlayPreparation::PreserveExisting, "fresh-overlay".len() as u64, ) @@ -7423,8 +7429,8 @@ mod tests { #[test] fn overlay_upper_path_targets_overlay_upperdir() { assert_eq!( - overlay_upper_path(GUEST_TLS_KEY_PATH), - "/upper/opt/openshell/tls/tls.key" + overlay_upper_path(GUEST_BOUNDARY_CONFIG_PATH), + "/upper/etc/openshell/boundary.json" ); } @@ -7450,6 +7456,24 @@ mod tests { assert_eq!(driver.capabilities().default_image, "openshell/sandbox:dev"); } + #[test] + fn host_control_receives_driver_owned_completion_marker() { + let mut command = Command::new("openshell-sandbox"); + configure_main_exit_marker(&mut command, Path::new("/private/sandboxes/sb-1")); + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + assert_eq!( + args, + [ + "--main-exit-marker".to_string(), + "/private/sandboxes/sb-1/main-process-exited".to_string(), + ] + ); + } + #[test] fn resolved_sandbox_image_prefers_template_image() { let driver = VmDriver { @@ -7638,7 +7662,7 @@ mod tests { } #[test] - fn build_guest_environment_sets_supervisor_defaults() { + fn build_guest_environment_sets_process_leaf_boot_metadata() { let config = VmDriverConfig { openshell_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() @@ -7650,16 +7674,18 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); + let env = build_guest_environment(&sandbox, &config); assert!(env.contains(&"HOME=/root".to_string())); - assert!(env.contains(&format!( - "OPENSHELL_ENDPOINT=http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080/" - ))); assert!(env.contains(&"OPENSHELL_SANDBOX_ID=sandbox-123".to_string())); assert!(env.contains(&"OPENSHELL_SANDBOX=breezy-rhinoceros".to_string())); - assert!(env.contains(&format!( - "OPENSHELL_SSH_SOCKET_PATH={GUEST_SSH_SOCKET_PATH}" - ))); + assert!( + !env.iter() + .any(|entry| entry.starts_with("OPENSHELL_ENDPOINT=")) + ); + assert!( + !env.iter() + .any(|entry| entry.starts_with("OPENSHELL_SSH_SOCKET_PATH=")) + ); } #[test] @@ -7673,78 +7699,45 @@ mod tests { } #[test] - fn persisted_legacy_sandbox_without_command_uses_scratch_main() { + fn build_guest_environment_keeps_user_values_in_child_channel() { let config = VmDriverConfig { openshell_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; - // Requests persisted before the canonical-main contract have a - // present DriverSandboxSpec but no command or tty fields. - let sandbox = Sandbox { - id: "legacy-sandbox".to_string(), - name: "legacy-sandbox".to_string(), - spec: Some(SandboxSpec::default()), - ..Default::default() - }; - - let env = build_guest_environment(&sandbox, &config, None); - let encoded = env - .iter() - .find_map(|entry| { - entry.strip_prefix(&format!( - "{}=", - openshell_core::sandbox_env::MAIN_PROCESS_SPEC - )) - }) - .expect("main process environment"); - let main = openshell_core::sandbox_env::MainProcessConfig::decode(encoded) - .expect("legacy persisted request should produce a valid main config"); - - assert_eq!( - main, - openshell_core::sandbox_env::MainProcessConfig::scratch() - ); - } - - #[test] - fn build_guest_environment_preserves_main_command_spaces() { - let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8080".to_string(), - ..Default::default() - }; - let command = vec![ - "sh".to_string(), - "-lc".to_string(), - "echo ready; while true; do sleep 1; done".to_string(), - ]; let sandbox = Sandbox { - id: "space-command".to_string(), - name: "space-command".to_string(), + id: "sandbox-123".to_string(), + name: "sandbox-123".to_string(), spec: Some(SandboxSpec { - command: command.clone(), + environment: HashMap::from([ + ("LD_PRELOAD".to_string(), "/workload/evil.so".to_string()), + ("BAD;touch /root/pwned".to_string(), "value".to_string()), + ]), ..Default::default() }), ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); - let encoded = env - .iter() - .find_map(|entry| { - entry.strip_prefix(&format!( - "{}=", - openshell_core::sandbox_env::MAIN_PROCESS_SPEC - )) - }) - .expect("main process environment"); + let env = build_guest_environment(&sandbox, &config); - assert!(!encoded.contains(char::is_whitespace)); - let main = openshell_core::sandbox_env::MainProcessConfig::decode(encoded).unwrap(); - assert_eq!(main.command, command); + assert!(!env.iter().any(|entry| entry.starts_with("LD_PRELOAD="))); + assert!(!env.iter().any(|entry| entry.starts_with("BAD;"))); + assert!( + !env.iter() + .any(|entry| { entry.starts_with(openshell_core::sandbox_env::USER_ENVIRONMENT) }) + ); + let child_env = merged_environment(&sandbox); + assert_eq!( + child_env.get("LD_PRELOAD"), + Some(&"/workload/evil.so".to_string()) + ); + assert_eq!( + child_env.get("BAD;touch /root/pwned"), + Some(&"value".to_string()) + ); } #[test] - fn build_guest_environment_uses_token_file_without_raw_token_env() { + fn build_guest_environment_excludes_all_gateway_credentials() { let config = VmDriverConfig { openshell_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() @@ -7763,16 +7756,16 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); + let env = build_guest_environment(&sandbox, &config); assert!(!env.iter().any(|v| v.starts_with(&format!( "{}=", openshell_core::sandbox_env::SANDBOX_TOKEN )))); - assert!(env.contains(&format!( - "{}={GUEST_SANDBOX_TOKEN_PATH}", + assert!(!env.iter().any(|v| v.starts_with(&format!( + "{}=", openshell_core::sandbox_env::SANDBOX_TOKEN_FILE - ))); + )))); } #[test] @@ -7794,7 +7787,7 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); + let env = build_guest_environment(&sandbox, &config); assert!( !env.iter().any(|v| v.starts_with(&format!( @@ -7831,7 +7824,7 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); + let env = build_guest_environment(&sandbox, &config); let telemetry_entries = env .iter() .filter(|entry| { @@ -7851,102 +7844,6 @@ mod tests { ); } - #[test] - fn build_guest_environment_clears_unsupported_network_capabilities() { - let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), - ..Default::default() - }; - let sandbox = Sandbox { - id: "sandbox-123".to_string(), - name: "sandbox-123".to_string(), - spec: Some(SandboxSpec { - environment: HashMap::from([( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), - )]), - ..Default::default() - }), - ..Default::default() - }; - let env = build_guest_environment(&sandbox, &config, None); - assert!(env.contains(&format!( - "{}=", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES - ))); - assert!(!env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - } - - #[test] - fn build_guest_environment_uses_endpoint_override_for_tap() { - let config = VmDriverConfig { - openshell_endpoint: "http://127.0.0.1:8080".to_string(), - ..Default::default() - }; - let sandbox = Sandbox { - id: "sandbox-123".to_string(), - name: "sandbox-123".to_string(), - spec: Some(SandboxSpec::default()), - ..Default::default() - }; - - let env = build_guest_environment(&sandbox, &config, Some("http://10.0.128.1:8080")); - assert!( - env.contains(&"OPENSHELL_ENDPOINT=http://10.0.128.1:8080".to_string()), - "TAP endpoint override must replace the default" - ); - let endpoint_count = env - .iter() - .filter(|e| e.starts_with("OPENSHELL_ENDPOINT=")) - .count(); - assert_eq!( - endpoint_count, 1, - "must have exactly one OPENSHELL_ENDPOINT" - ); - } - - #[test] - fn guest_visible_openshell_endpoint_rewrites_loopback_hosts_to_gvproxy_host_alias() { - assert_eq!( - guest_visible_openshell_endpoint("http://127.0.0.1:8080"), - format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080/") - ); - assert_eq!( - guest_visible_openshell_endpoint("http://localhost:8080"), - format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080/") - ); - assert_eq!( - guest_visible_openshell_endpoint("https://[::1]:8443"), - format!("https://{GVPROXY_HOST_LOOPBACK_ALIAS}:8443/") - ); - } - - #[test] - fn guest_visible_openshell_endpoint_preserves_non_loopback_hosts() { - assert_eq!( - guest_visible_openshell_endpoint(&format!( - "http://{OPENSHELL_HOST_GATEWAY_ALIAS}:8080" - )), - format!("http://{OPENSHELL_HOST_GATEWAY_ALIAS}:8080") - ); - assert_eq!( - guest_visible_openshell_endpoint(&format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080")), - format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080") - ); - assert_eq!( - guest_visible_openshell_endpoint("http://192.168.127.1:8080"), - "http://192.168.127.1:8080" - ); - assert_eq!( - guest_visible_openshell_endpoint("https://gateway.internal:8443"), - "https://gateway.internal:8443" - ); - } - #[test] fn image_reference_registry_host_defaults_to_docker_hub() { assert_eq!(image_reference_registry_host("ubuntu:24.04"), "docker.io"); @@ -8084,7 +7981,7 @@ mod tests { } #[test] - fn build_guest_environment_includes_tls_paths_for_https_endpoint() { + fn build_guest_environment_keeps_tls_paths_host_side() { let config = VmDriverConfig { openshell_endpoint: "https://127.0.0.1:8443".to_string(), guest_tls_ca: Some(PathBuf::from("/host/ca.crt")), @@ -8099,10 +7996,8 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); - assert!(env.contains(&format!("OPENSHELL_TLS_CA={GUEST_TLS_CA_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_CERT={GUEST_TLS_CERT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_KEY={GUEST_TLS_KEY_PATH}"))); + let env = build_guest_environment(&sandbox, &config); + assert!(!env.iter().any(|entry| entry.starts_with("OPENSHELL_TLS_"))); } #[test] @@ -8167,6 +8062,7 @@ mod tests { record.state_dir = retry_state_dir; record.process = Some(Arc::new(Mutex::new(VmProcess { child: spawn_exited_child(), + supervisor: spawn_exited_child(), deleting: false, }))); } @@ -8362,6 +8258,58 @@ mod tests { .expect("dns endpoint should be accepted"); } + #[test] + fn host_control_endpoint_rewrites_guest_host_aliases() { + for host in LEGACY_HOST_GATEWAY_ALIASES { + assert_eq!( + host_control_openshell_endpoint(&format!("https://{host}:8443/control")) + .expect("guest alias should be rewritten"), + "https://127.0.0.1:8443/control", + "host alias {host}" + ); + } + } + + #[test] + fn host_control_endpoint_preserves_remote_gateway() { + assert_eq!( + host_control_openshell_endpoint("https://gateway.internal:8443") + .expect("remote gateway should be preserved"), + "https://gateway.internal:8443/" + ); + } + + #[test] + fn relative_state_dir_is_resolved_from_the_working_directory() { + let working_dir = std::env::current_dir().expect("working directory"); + assert_eq!( + absolute_state_dir(Path::new("target/driver-state")).expect("resolve state dir"), + working_dir.join("target/driver-state") + ); + + let absolute = working_dir.join("existing-absolute-state"); + assert_eq!( + absolute_state_dir(&absolute).expect("preserve absolute state dir"), + absolute + ); + } + + #[test] + fn host_control_environment_contains_only_explicit_values() { + let mut command = Command::new("openshell-sandbox"); + command.env("UNTRUSTED_PARENT_VALUE", "must-not-leak"); + isolate_host_control_environment(&mut command); + command.env("DRIVER_OWNED_VALUE", "kept"); + + let environment = command.as_std().get_envs().collect::>(); + assert_eq!(environment.len(), 1); + assert_eq!(environment[0].0, "DRIVER_OWNED_VALUE"); + assert_eq!( + environment[0].1.and_then(std::ffi::OsStr::to_str), + Some("kept") + ); + } + #[test] fn prepared_image_cache_identity_includes_rootfs_layout_and_openshell_version() { assert_eq!( @@ -8374,14 +8322,14 @@ mod tests { } #[test] - fn bootstrap_image_cache_identity_includes_rootfs_layout_and_openshell_version() { - assert_eq!( - bootstrap_image_cache_identity("sha256:bootstrap-image"), - format!( - "sandbox-bootstrap-rootfs-ext4-v3:openshell-{}:sha256:bootstrap-image", - openshell_core::VERSION - ) - ); + fn bootstrap_image_cache_identity_includes_rootfs_layout_version_and_guest_runtime() { + let identity = bootstrap_image_cache_identity("sha256:bootstrap-image"); + assert!(identity.starts_with(&format!( + "sandbox-bootstrap-rootfs-ext4-v4:openshell-{}:guest-", + openshell_core::VERSION + ))); + assert!(identity.ends_with(":sha256:bootstrap-image")); + assert!(identity.contains(&sandbox_guest_runtime_identity())); } #[test] @@ -8481,66 +8429,6 @@ mod tests { ); } - #[tokio::test] - async fn read_guest_tls_materials_reports_missing_input() { - let base = unique_temp_dir(); - let source_dir = base.join("missing-source"); - - let err = read_guest_tls_materials(&VmDriverTlsPaths { - ca: source_dir.join("ca.crt"), - cert: source_dir.join("tls.crt"), - key: source_dir.join("tls.key"), - }) - .await - .expect_err("missing TLS materials should fail before image injection"); - - assert!(err.contains("ca.crt")); - - let _ = std::fs::remove_dir_all(base); - } - - #[cfg(unix)] - #[test] - fn stage_guest_tls_materials_places_files_in_overlay_upper_with_private_key_mode() { - use std::os::unix::fs::PermissionsExt as _; - - let base = unique_temp_dir(); - let materials = GuestTlsMaterials { - ca: b"ca".to_vec(), - cert: b"cert".to_vec(), - key: b"key".to_vec(), - }; - - stage_guest_tls_materials(&base, &materials).expect("stage TLS materials"); - - assert_eq!( - fs::read( - base.join("upper") - .join(GUEST_TLS_CA_PATH.trim_start_matches('/')) - ) - .unwrap(), - b"ca" - ); - assert_eq!( - fs::read( - base.join("upper") - .join(GUEST_TLS_CERT_PATH.trim_start_matches('/')) - ) - .unwrap(), - b"cert" - ); - let key_path = base - .join("upper") - .join(GUEST_TLS_KEY_PATH.trim_start_matches('/')); - assert_eq!(fs::read(&key_path).unwrap(), b"key"); - assert_eq!( - fs::metadata(&key_path).unwrap().permissions().mode() & 0o777, - 0o600 - ); - - let _ = std::fs::remove_dir_all(base); - } - #[test] fn subnet_allocator_assigns_and_releases() { let mut alloc = SubnetAllocator::new(Ipv4Addr::new(10, 0, 128, 0), 17); @@ -8615,6 +8503,7 @@ mod tests { }; let process = Arc::new(Mutex::new(VmProcess { child, + supervisor: spawn_exited_child(), deleting: false, })); @@ -8764,11 +8653,11 @@ mod tests { assert_eq!(plan.vcpus, 8); assert_eq!(plan.mem_mib, 16384); assert_eq!(plan.gpu_bdf.as_deref(), Some("0000:01:00.0")); - assert!(plan.tap_device.is_some()); - assert!(plan.guest_ip.is_some()); - assert!(plan.host_ip.is_some()); + assert!(plan.tap_device.is_none()); + assert!(plan.guest_ip.is_none()); + assert!(plan.host_ip.is_none()); assert!(plan.vsock_cid.is_some()); - assert!(plan.guest_mac.is_some()); + assert!(plan.guest_mac.is_none()); } #[test] @@ -8820,13 +8709,11 @@ mod tests { .expect("backend feature should resolve"); assert_eq!(plan.backend, VmBackend::Qemu); - assert!(plan.tap_device.is_some()); - assert!(plan.guest_ip.is_some()); - assert!(plan.host_ip.is_some()); + assert!(plan.tap_device.is_none()); + assert!(plan.guest_ip.is_none()); + assert!(plan.host_ip.is_none()); assert!(plan.vsock_cid.is_some()); - assert!(plan.guest_mac.is_some()); - - driver.release_subnet("sandbox-vfio"); + assert!(plan.guest_mac.is_none()); } #[test] @@ -8842,11 +8729,10 @@ mod tests { .expect("backend requirement should resolve"); assert_eq!(plan.backend, VmBackend::Qemu); - assert!(plan.tap_device.is_some()); - assert!(plan.guest_ip.is_some()); - assert!(plan.host_ip.is_some()); - - driver.release_subnet("sandbox-qemu"); + assert!(plan.tap_device.is_none()); + assert!(plan.guest_ip.is_none()); + assert!(plan.host_ip.is_none()); + assert!(plan.vsock_cid.is_some()); } #[test] @@ -9079,7 +8965,7 @@ mod tests { } #[test] - fn upstream_proxy_args_pass_guest_paths_not_host_paths() { + fn upstream_proxy_args_pass_host_paths_to_host_control() { let config = proxy_config( Some("http://proxy.corp.test:3128"), Some("/etc/openshell/secrets/proxy-auth"), @@ -9087,24 +8973,18 @@ mod tests { ); let args = upstream_proxy_cli_args(&config); - // The credential and CA live at fixed guest paths; the gateway-host - // paths the operator configured must never reach the guest argv. + // Control runs on the gateway host and receives the operator-owned + // paths directly; neither path is copied into the guest. let auth = args .iter() .position(|arg| arg == "--upstream-proxy-auth-file") .map(|i| args[i + 1].as_str()); - assert_eq!(auth, Some(GUEST_UPSTREAM_PROXY_AUTH_PATH)); + assert_eq!(auth, Some("/etc/openshell/secrets/proxy-auth")); let ca = args .iter() .position(|arg| arg == "--upstream-proxy-ca-bundle") .map(|i| args[i + 1].as_str()); - assert_eq!(ca, Some(GUEST_PROXY_CA_PATH)); - assert!( - !args - .iter() - .any(|arg| arg.contains("/etc/openshell/secrets") || arg.contains("corp-ca.pem")), - "host paths leaked into the guest argv: {args:?}" - ); + assert_eq!(ca, Some("/etc/openshell/tls/corp-ca.pem")); } #[test] @@ -9312,32 +9192,18 @@ mod tests { } #[test] - fn qemu_launch_plan_rejects_a_proxy_at_the_allocated_tap_host() { - // The preflight has to run against the address this sandbox actually - // got, which only exists once the launch plan's subnet is allocated. - // A proxy there is what `host.openshell.internal` resolves to in the - // guest, and the driver's own nftables input chain drops the port. - let probe = test_driver_with_extensions(LifecycleExtensionRegistry::new()); - let tap_host = probe - .build_vm_launch_plan("sandbox-proxy-tap", true, true, None) - .expect("gpu plan should build") - .host_ip - .expect("a QEMU plan carries a TAP host address"); - probe.release_subnet("sandbox-proxy-tap"); - - let driver = test_driver_with_proxy(&format!("http://{tap_host}:8080")); + fn qemu_launch_plan_uses_vsock_only_with_host_proxy() { + let driver = test_driver_with_proxy("http://127.0.0.1:8080"); let mut plan = driver .build_vm_launch_plan("sandbox-proxy-tap", true, true, None) .expect("gpu plan should build"); - assert_eq!(plan.host_ip.as_deref(), Some(tap_host.as_str())); - - let err = driver + driver .resolve_launch_plan_backend("sandbox-proxy-tap", true, None, &mut plan) - .expect_err("a proxy at the TAP host address is unreachable from the guest"); - assert_eq!(err.code(), Code::FailedPrecondition); - assert!(err.message().contains(&tap_host), "{err}"); - - driver.release_subnet("sandbox-proxy-tap"); + .expect("host control can reach a host-loopback proxy"); + assert!(plan.vsock_cid.is_some()); + assert!(plan.tap_device.is_none()); + assert!(plan.host_ip.is_none()); + assert!(plan.guest_ip.is_none()); } #[test] @@ -9414,7 +9280,7 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); + let env = build_guest_environment(&sandbox, &config); assert!( !env.iter().any(|entry| entry.starts_with("--upstream")), "driver environment must never carry supervisor arguments: {env:?}" diff --git a/crates/openshell-driver-vm/src/ffi.rs b/crates/openshell-driver-vm/src/ffi.rs index 423ad6f05b..f84ea35743 100644 --- a/crates/openshell-driver-vm/src/ffi.rs +++ b/crates/openshell-driver-vm/src/ffi.rs @@ -52,23 +52,8 @@ type KrunSetConsoleOutput = unsafe extern "C" fn(ctx_id: u32, filepath: *const c type KrunStartEnter = unsafe extern "C" fn(ctx_id: u32) -> i32; type KrunDisableImplicitVsock = unsafe extern "C" fn(ctx_id: u32) -> i32; type KrunAddVsock = unsafe extern "C" fn(ctx_id: u32, tsi_features: u32) -> i32; -#[cfg(target_os = "macos")] -type KrunAddNetUnixgram = unsafe extern "C" fn( - ctx_id: u32, - c_path: *const c_char, - fd: i32, - c_mac: *const u8, - features: u32, - flags: u32, -) -> i32; -type KrunAddNetUnixstream = unsafe extern "C" fn( - ctx_id: u32, - c_path: *const c_char, - fd: i32, - c_mac: *const u8, - features: u32, - flags: u32, -) -> i32; +type KrunAddVsockPort2 = + unsafe extern "C" fn(ctx_id: u32, port: u32, filepath: *const c_char, listen: bool) -> i32; // Field names mirror the libkrun C API symbol names (`krun_*`); preserving // the prefix keeps the FFI binding 1:1 with the upstream library. @@ -86,10 +71,7 @@ pub struct LibKrun { pub krun_start_enter: KrunStartEnter, pub krun_disable_implicit_vsock: KrunDisableImplicitVsock, pub krun_add_vsock: KrunAddVsock, - #[cfg(target_os = "macos")] - pub krun_add_net_unixgram: KrunAddNetUnixgram, - #[allow(dead_code)] // Used on Linux when gvproxy runs in qemu/unixstream mode. - pub krun_add_net_unixstream: KrunAddNetUnixstream, + pub krun_add_vsock_port2: KrunAddVsockPort2, } static LIBKRUN: OnceLock = OnceLock::new(); @@ -151,13 +133,7 @@ impl LibKrun { &libkrun_path, )?, krun_add_vsock: load_symbol(library, b"krun_add_vsock\0", &libkrun_path)?, - #[cfg(target_os = "macos")] - krun_add_net_unixgram: load_symbol(library, b"krun_add_net_unixgram\0", &libkrun_path)?, - krun_add_net_unixstream: load_symbol( - library, - b"krun_add_net_unixstream\0", - &libkrun_path, - )?, + krun_add_vsock_port2: load_symbol(library, b"krun_add_vsock_port2\0", &libkrun_path)?, }) } } diff --git a/crates/openshell-driver-vm/src/isolation/mod.rs b/crates/openshell-driver-vm/src/isolation/mod.rs new file mode 100644 index 0000000000..caba5f68ea --- /dev/null +++ b/crates/openshell-driver-vm/src/isolation/mod.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! VM provisioning for the shared authenticated boundary protocol. +//! +//! This module deliberately contains no lifecycle, process, network, or wire +//! implementation. The driver chooses the host transport and binds immutable +//! VM claims; `openshell-isolation-interface` and `openshell-sandbox` provide +//! the common control and boundary behavior. + +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; + +use openshell_isolation_interface::boundary_protocol::{ + BOUNDARY_PROTOCOL_VERSION, BoundaryAgentIdentity, BoundaryConfig, BoundaryListener, + BoundaryTopology, BoundaryTransport, +}; + +/// Driver-owned inputs that bind one VM generation to one supervisor boundary. +pub struct VmBoundarySpec { + pub boundary_id: String, + pub bootstrap_token: String, + pub generation: String, + pub image_identity: String, + pub transport: BoundaryTransport, + pub control_port: u32, + pub agent_uid: u32, + pub agent_gid: u32, + pub trusted_runtime_root: PathBuf, + pub child_env: HashMap, +} + +/// The protected guest config and matching host descriptor for one VM. +pub struct VmBoundaryProvisioning { + pub boundary_config: BoundaryConfig, + pub topology: BoundaryTopology, +} + +impl VmBoundarySpec { + /// Produce both sides of the common protocol from one set of immutable + /// driver inputs so their identity claims cannot drift. + #[must_use] + pub fn provision(self) -> VmBoundaryProvisioning { + let resource_claims = BTreeMap::from([ + ("vm.generation".to_string(), self.generation), + ("vm.image_identity".to_string(), self.image_identity), + ]); + VmBoundaryProvisioning { + boundary_config: BoundaryConfig { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: self.boundary_id.clone(), + bootstrap_token: self.bootstrap_token.clone(), + listener: BoundaryListener::Vsock { + control_port: self.control_port, + }, + resource_claims: resource_claims.clone(), + agent_identity: BoundaryAgentIdentity::Resolved { + uid: self.agent_uid, + gid: self.agent_gid, + }, + protect_config_file: false, + trusted_runtime_root: self.trusted_runtime_root, + child_env: self.child_env, + }, + topology: BoundaryTopology { + protocol_version: BOUNDARY_PROTOCOL_VERSION, + boundary_id: self.boundary_id, + transport: self.transport, + // The host-side control process is the network broker, so + // reserved host aliases terminate at its loopback address + // after crossing the authenticated boundary channel. + host_gateway_ip: Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + resource_claims, + bootstrap_token: self.bootstrap_token, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provisioning_binds_identical_resource_claims() { + let provisioned = VmBoundarySpec { + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "a".repeat(64), + generation: "generation-1".to_string(), + image_identity: "sha256:image".to_string(), + transport: BoundaryTransport::Vsock { + guest_cid: 42, + control_port: 5500, + }, + control_port: 5500, + agent_uid: 1000, + agent_gid: 1000, + trusted_runtime_root: PathBuf::from("/opt/openshell/runtime"), + child_env: HashMap::new(), + } + .provision(); + + assert_eq!( + provisioned.boundary_config.resource_claims, + provisioned.topology.resource_claims + ); + assert_eq!( + provisioned.topology.resource_claims["vm.generation"], + "generation-1" + ); + assert_eq!( + provisioned.topology.host_gateway_ip, + Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)) + ); + } +} diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs index f34c7dda8d..8318ae3a4f 100644 --- a/crates/openshell-driver-vm/src/lib.rs +++ b/crates/openshell-driver-vm/src/lib.rs @@ -11,24 +11,39 @@ compile_error!( build a telemetry-free VM driver with `--no-default-features --features defaults-without-telemetry`" ); +#[cfg(feature = "compute-driver")] pub mod driver; +#[cfg(feature = "compute-driver")] mod embedded_runtime; +#[cfg(feature = "compute-driver")] mod ffi; +#[cfg(feature = "compute-driver")] pub mod gpu; +#[cfg(feature = "compute-driver")] +mod isolation; +#[cfg(feature = "compute-driver")] pub mod lifecycle; +#[cfg(feature = "compute-driver")] mod nft_ruleset; +#[cfg(feature = "compute-driver")] pub mod otel_tracing; +#[cfg(feature = "compute-driver")] pub mod procguard; +#[cfg(feature = "compute-driver")] mod rootfs; +#[cfg(feature = "compute-driver")] mod runtime; +#[cfg(feature = "compute-driver")] pub use driver::{VmDriver, VmDriverConfig}; +#[cfg(feature = "compute-driver")] pub use lifecycle::{ BackendFeature, ExtensionCapabilities, ExtensionDescriptor, GuestInitDropin, LaunchAbortReason, LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult, RestoreContext, }; +#[cfg(feature = "compute-driver")] pub use runtime::{ - VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces, + VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, VsockPortMap, cleanup_stale_tap_interfaces, configured_runtime_dir, run_vm, }; diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 2546cb2606..60f5aa967e 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -9,7 +9,9 @@ use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServ use openshell_driver_vm::otel_tracing::compute_driver_rpc_layer; #[cfg(target_os = "macos")] use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; -use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm}; +use openshell_driver_vm::{ + VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, VsockPortMap, procguard, run_vm, +}; use std::io; use std::net::SocketAddr; use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; @@ -193,6 +195,12 @@ struct Args { #[arg(long, hide = true)] vm_gateway_port: Option, + + #[arg(long, hide = true)] + vm_vsock_control_port: Option, + + #[arg(long, hide = true)] + vm_vsock_control_socket: Option, } #[tokio::main] @@ -586,6 +594,23 @@ fn build_vm_launch_config(args: &Args) -> std::result::Result Some(VsockPortMap { + guest_port, + host_socket, + host_initiated: true, + }), + (None, None) => None, + _ => { + return Err( + "--vm-vsock-control-port and --vm-vsock-control-socket must be set together" + .to_string(), + ); + } + }, }) } diff --git a/crates/openshell-driver-vm/src/nft_ruleset.rs b/crates/openshell-driver-vm/src/nft_ruleset.rs index fe3e86c902..aae6cf0506 100644 --- a/crates/openshell-driver-vm/src/nft_ruleset.rs +++ b/crates/openshell-driver-vm/src/nft_ruleset.rs @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::fmt::Write; - /// Sanitize a TAP device name for use as an nftables table name suffix. /// Assumes device names match `vmtap-[a-f0-9]+` (driver-controlled). fn sanitize_table_name(device: &str) -> String { @@ -14,74 +12,16 @@ pub fn teardown_table_name(device: &str) -> String { format!("openshell_vm_{}", sanitize_table_name(device)) } -/// Generate the nftables ruleset for VM TAP networking. -pub fn generate_tap_ruleset(tap_device: &str, subnet: &str, gateway_port: u16) -> String { - let table_name = teardown_table_name(tap_device); - let mut ruleset = String::with_capacity(512); - - writeln!(ruleset, "table ip {table_name} {{").unwrap(); - writeln!(ruleset, " chain postrouting {{").unwrap(); - writeln!( - ruleset, - " type nat hook postrouting priority 100; policy accept;" - ) - .unwrap(); - writeln!(ruleset, " ip saddr {subnet} masquerade").unwrap(); - writeln!(ruleset, " }}").unwrap(); - writeln!(ruleset, " chain forward {{").unwrap(); - writeln!( - ruleset, - " type filter hook forward priority 0; policy accept;" - ) - .unwrap(); - writeln!(ruleset, " iifname \"{tap_device}\" accept").unwrap(); - writeln!( - ruleset, - " oifname \"{tap_device}\" ct state related,established accept" - ) - .unwrap(); - writeln!(ruleset, " oifname \"{tap_device}\" drop").unwrap(); - writeln!(ruleset, " }}").unwrap(); - writeln!(ruleset, " chain input {{").unwrap(); - writeln!( - ruleset, - " type filter hook input priority 0; policy accept;" - ) - .unwrap(); - writeln!( - ruleset, - " iifname \"{tap_device}\" tcp dport {gateway_port} accept" - ) - .unwrap(); - writeln!(ruleset, " iifname \"{tap_device}\" drop").unwrap(); - writeln!(ruleset, " }}").unwrap(); - writeln!(ruleset, "}}").unwrap(); - - ruleset -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn generates_tap_setup_ruleset() { - let ruleset = generate_tap_ruleset("vmtap-abcd", "10.0.128.0/30", 8080); - assert!(ruleset.contains("table ip openshell_vm_vmtap_abcd {")); - assert!(ruleset.contains("type nat hook postrouting priority 100; policy accept;")); - assert!(ruleset.contains("ip saddr 10.0.128.0/30 masquerade")); - assert!(ruleset.contains("type filter hook forward priority 0; policy accept;")); - assert!(ruleset.contains("iifname \"vmtap-abcd\" accept")); - assert!(ruleset.contains("oifname \"vmtap-abcd\" ct state related,established accept")); - assert!(ruleset.contains("oifname \"vmtap-abcd\" drop")); - assert!(ruleset.contains("type filter hook input priority 0; policy accept;")); - assert!(ruleset.contains("iifname \"vmtap-abcd\" tcp dport 8080 accept")); - } - #[test] fn table_name_sanitizes_device_name() { - let ruleset = generate_tap_ruleset("vmtap-abc-123", "10.0.128.0/30", 8080); - assert!(ruleset.contains("table ip openshell_vm_vmtap_abc_123 {")); + assert_eq!( + teardown_table_name("vmtap-abc-123"), + "openshell_vm_vmtap_abc_123" + ); } #[test] diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 9046913c9d..588d34a53f 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use sha2::{Digest, Sha256}; use std::fs; use std::fs::File; #[cfg(test)] @@ -8,9 +9,13 @@ use std::io::BufWriter; use std::io::{Cursor, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::process::Command; +#[cfg(target_os = "linux")] +use std::sync::OnceLock; use std::sync::atomic::{AtomicU64, Ordering}; const SUPERVISOR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/openshell-sandbox.zst")); +const SUPERVISOR_RUNTIME: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/openshell-runtime.tar.zst")); const UMOCI: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/umoci.zst")); const ROOTFS_VARIANT_MARKER: &str = ".openshell-rootfs-variant"; const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh"; @@ -18,6 +23,7 @@ const SANDBOX_SUPERVISOR_PATH: &str = openshell_core::driver_utils::SUPERVISOR_C const SANDBOX_UMOCI_PATH: &str = openshell_core::container_paths::VM_UMOCI_PATH; const SANDBOX_OWNER_NORMALIZED_MARKER: &str = openshell_core::container_paths::VM_SANDBOX_OWNER_NORMALIZED_MARKER; +const SANDBOX_SUPERVISOR_RUNTIME_PATH: &str = "/opt/openshell/bin/openshell-runtime"; const ROOTFS_IMAGE_MIN_SIZE_BYTES: u64 = 512 * 1024 * 1024; const ROOTFS_IMAGE_MIN_HEADROOM_BYTES: u64 = 256 * 1024 * 1024; const EXT4_IMAGE_MIN_HEADROOM_BYTES: u64 = 16 * 1024 * 1024; @@ -27,6 +33,149 @@ pub const fn sandbox_guest_init_path() -> &'static str { SANDBOX_GUEST_INIT_PATH } +/// Identity of every embedded artifact materialized into a bootstrap rootfs. +/// +/// Including this in the image-cache key makes local, uncommitted guest-leaf +/// changes invalidate the cache even when the `OpenShell` version is unchanged. +pub fn sandbox_guest_runtime_identity() -> String { + let mut hasher = Sha256::new(); + hasher.update(SUPERVISOR); + hasher.update(SUPERVISOR_RUNTIME); + hasher.update(UMOCI); + hasher.update(include_bytes!("../scripts/openshell-vm-sandbox-init.sh")); + format!("{:x}", hasher.finalize()) +} + +/// Materialize the supervisor embedded in the VM driver for host-side use. +#[cfg(target_os = "linux")] +pub fn extract_host_supervisor(path: &Path) -> Result<(), String> { + if SUPERVISOR.is_empty() { + return Err( + "host supervisor is not embedded; run `mise run vm:supervisor` and rebuild openshell-driver-vm" + .to_string(), + ); + } + let supervisor = embedded_host_supervisor()?; + install_host_supervisor_atomically(path, &supervisor)?; + validate_host_supervisor(path) +} + +#[cfg(target_os = "linux")] +pub fn validate_host_supervisor(path: &Path) -> Result<(), String> { + validate_host_supervisor_digest(path, embedded_host_supervisor_digest()?) +} + +#[cfg(target_os = "linux")] +fn validate_host_supervisor_digest(path: &Path, expected: [u8; 32]) -> Result<(), String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("inspect cached host supervisor {}: {error}", path.display()))?; + if !metadata.file_type().is_file() { + return Err(format!( + "cached host supervisor is not a regular file: {}", + path.display() + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + if metadata.permissions().mode() & 0o111 == 0 { + return Err(format!( + "cached host supervisor is not executable: {}", + path.display() + )); + } + } + let actual = sha256_reader( + File::open(path) + .map_err(|error| format!("open cached host supervisor {}: {error}", path.display()))?, + ) + .map_err(|error| format!("hash cached host supervisor {}: {error}", path.display()))?; + if actual != expected { + return Err(format!( + "cached host supervisor content does not match embedded runtime: {}", + path.display() + )); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn embedded_host_supervisor() -> Result, String> { + zstd::decode_all(Cursor::new(SUPERVISOR)) + .map_err(|error| format!("decompress host supervisor: {error}")) +} + +#[cfg(target_os = "linux")] +fn embedded_host_supervisor_digest() -> Result<[u8; 32], String> { + static DIGEST: OnceLock> = OnceLock::new(); + DIGEST + .get_or_init(|| embedded_host_supervisor().map(|bytes| sha256_bytes(&bytes))) + .clone() +} + +#[cfg(target_os = "linux")] +fn sha256_bytes(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} + +#[cfg(target_os = "linux")] +fn sha256_reader(mut reader: impl Read) -> std::io::Result<[u8; 32]> { + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 16 * 1024]; + loop { + let read = reader.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(hasher.finalize().into()) +} + +#[cfg(target_os = "linux")] +fn install_host_supervisor_atomically(path: &Path, bytes: &[u8]) -> Result<(), String> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .ok_or_else(|| format!("host supervisor path has no parent: {}", path.display()))?; + fs::create_dir_all(parent).map_err(|error| format!("create {}: {error}", parent.display()))?; + let temporary = parent.join(format!( + ".openshell-sandbox.tmp-{}-{}", + std::process::id(), + INJECTION_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + let result = (|| { + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o755); + } + let mut file = options + .open(&temporary) + .map_err(|error| format!("create {}: {error}", temporary.display()))?; + file.write_all(bytes) + .map_err(|error| format!("write {}: {error}", temporary.display()))?; + file.sync_all() + .map_err(|error| format!("sync {}: {error}", temporary.display()))?; + fs::rename(&temporary, path).map_err(|error| { + format!( + "commit cached host supervisor {} to {}: {error}", + temporary.display(), + path.display() + ) + })?; + File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|error| format!("sync host supervisor cache {}: {error}", parent.display())) + })(); + if result.is_err() { + let _ = fs::remove_file(&temporary); + } + result +} + #[allow(clippy::similar_names)] pub fn prepare_sandbox_rootfs_from_image_root( rootfs: &Path, @@ -376,6 +525,8 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> } ensure_supervisor_binary(rootfs)?; + ensure_supervisor_runtime(rootfs)?; + ensure_guest_init_ip(rootfs)?; ensure_umoci_binary(rootfs)?; let opt_dir = rootfs.join("opt/openshell"); @@ -392,9 +543,55 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> Ok(()) } +fn ensure_guest_init_ip(rootfs: &Path) -> Result<(), String> { + const IP_PATHS: [&str; 4] = ["sbin/ip", "usr/sbin/ip", "bin/ip", "usr/bin/ip"]; + if IP_PATHS.iter().any(|path| rootfs.join(path).is_file()) { + return Ok(()); + } + + // Guest init runs before the process leaf can enter its trusted helper + // runtime. Images such as stock Ubuntu do not ship iproute2, so install a + // driver-owned launcher that executes the embedded musl helper explicitly. + // The helper and loader are both materialized from the trusted runtime, + // never from the workload image. + let path = rootfs.join("usr/sbin/ip"); + let parent = path + .parent() + .ok_or_else(|| format!("guest ip launcher path has no parent: {}", path.display()))?; + fs::create_dir_all(parent).map_err(|error| format!("create {}: {error}", parent.display()))?; + fs::write( + &path, + r#"#!/bin/sh +set -eu +runtime=/opt/openshell/bin/openshell-runtime +for loader in "$runtime"/lib/ld-musl-*.so.1; do + if [ -x "$loader" ]; then + for helper in "$runtime"/sbin/ip "$runtime"/usr/sbin/ip "$runtime"/bin/ip "$runtime"/usr/bin/ip; do + if [ -x "$helper" ]; then + exec "$loader" --library-path "$runtime/lib:$runtime/usr/lib" "$helper" "$@" + fi + done + fi +done +echo "trusted OpenShell ip helper is unavailable" >&2 +exit 127 +"#, + ) + .map_err(|error| format!("write {}: {error}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)) + .map_err(|error| format!("chmod {}: {error}", path.display()))?; + } + Ok(()) +} + pub fn validate_sandbox_rootfs(rootfs: &Path) -> Result<(), String> { require_rootfs_path(rootfs, SANDBOX_GUEST_INIT_PATH)?; require_rootfs_path(rootfs, SANDBOX_SUPERVISOR_PATH)?; + validate_supervisor_runtime(rootfs)?; require_rootfs_path(rootfs, SANDBOX_UMOCI_PATH)?; require_any_rootfs_path(rootfs, &["/bin/bash"])?; require_any_rootfs_path(rootfs, &["/bin/mount", "/usr/bin/mount"])?; @@ -795,20 +992,20 @@ fn ensure_sandbox_guest_user( let etc_dir = rootfs.join("etc"); fs::create_dir_all(&etc_dir).map_err(|e| format!("create {}: {e}", etc_dir.display()))?; - ensure_line_in_file( + replace_or_append_line( &etc_dir.join("group"), &format!("sandbox:x:{sandbox_gid}:"), |line| line.starts_with("sandbox:"), )?; - ensure_line_in_file(&etc_dir.join("gshadow"), "sandbox:!::", |line| { + replace_or_append_line(&etc_dir.join("gshadow"), "sandbox:!::", |line| { line.starts_with("sandbox:") })?; - ensure_line_in_file( + replace_or_append_line( &etc_dir.join("passwd"), &format!("sandbox:x:{sandbox_uid}:{sandbox_gid}:OpenShell Sandbox:/sandbox:/bin/bash"), |line| line.starts_with("sandbox:"), )?; - ensure_line_in_file( + replace_or_append_line( &etc_dir.join("shadow"), "sandbox:!:20123:0:99999:7:::", |line| line.starts_with("sandbox:"), @@ -817,28 +1014,36 @@ fn ensure_sandbox_guest_user( Ok(()) } -fn ensure_line_in_file( +fn replace_or_append_line( path: &Path, line: &str, - exists: impl Fn(&str) -> bool, + matches: impl Fn(&str) -> bool, ) -> Result<(), String> { - let mut contents = if path.exists() { + let contents = if path.exists() { fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))? } else { String::new() }; - - if contents.lines().any(exists) { - return Ok(()); + let mut output = String::with_capacity(contents.len().max(line.len() + 1)); + let mut replaced = false; + for existing in contents.lines() { + if matches(existing) { + if replaced { + continue; + } + output.push_str(line); + replaced = true; + } else { + output.push_str(existing); + } + output.push('\n'); } - - if !contents.is_empty() && !contents.ends_with('\n') { - contents.push('\n'); + if !replaced { + output.push_str(line); + output.push('\n'); } - contents.push_str(line); - contents.push('\n'); - fs::write(path, contents).map_err(|e| format!("write {}: {e}", path.display())) + fs::write(path, output).map_err(|e| format!("write {}: {e}", path.display())) } fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> { @@ -871,6 +1076,84 @@ fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> { Ok(()) } +fn ensure_supervisor_runtime(rootfs: &Path) -> Result<(), String> { + if SUPERVISOR_RUNTIME.is_empty() { + return validate_supervisor_runtime(rootfs).map_err(|_| { + "trusted supervisor helper runtime not embedded. Build openshell-driver-vm with OPENSHELL_VM_RUNTIME_COMPRESSED_DIR set and run `mise run vm:supervisor` first" + .to_string() + }); + } + + install_supervisor_runtime_archive(rootfs, SUPERVISOR_RUNTIME) +} + +fn install_supervisor_runtime_archive(rootfs: &Path, archive_bytes: &[u8]) -> Result<(), String> { + let destination = rootfs.join("opt/openshell/bin"); + fs::create_dir_all(&destination) + .map_err(|e| format!("create {}: {e}", destination.display()))?; + let runtime = rootfs.join(SANDBOX_SUPERVISOR_RUNTIME_PATH.trim_start_matches('/')); + match fs::symlink_metadata(&runtime) { + Ok(metadata) if metadata.file_type().is_dir() => fs::remove_dir_all(&runtime) + .map_err(|e| format!("remove untrusted runtime {}: {e}", runtime.display()))?, + Ok(_) => fs::remove_file(&runtime) + .map_err(|e| format!("remove untrusted runtime {}: {e}", runtime.display()))?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("inspect runtime {}: {error}", runtime.display())), + } + let decoder = zstd::Decoder::new(Cursor::new(archive_bytes)) + .map_err(|e| format!("decompress supervisor runtime: {e}"))?; + let mut archive = tar::Archive::new(decoder); + for entry in archive + .entries() + .map_err(|e| format!("open supervisor runtime archive: {e}"))? + { + let mut entry = entry.map_err(|e| format!("read supervisor runtime archive: {e}"))?; + let kind = entry.header().entry_type(); + if !kind.is_file() && !kind.is_dir() { + return Err( + "supervisor runtime archive contains a non-materialized link or special file" + .to_string(), + ); + } + if !entry + .unpack_in(&destination) + .map_err(|e| format!("extract supervisor runtime archive: {e}"))? + { + return Err("supervisor runtime archive contains a path outside its root".to_string()); + } + } + validate_supervisor_runtime(rootfs) +} + +fn validate_supervisor_runtime(rootfs: &Path) -> Result<(), String> { + let runtime = rootfs.join(SANDBOX_SUPERVISOR_RUNTIME_PATH.trim_start_matches('/')); + let has_ip = ["sbin/ip", "usr/sbin/ip", "bin/ip", "usr/bin/ip"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_nft = ["sbin/nft", "usr/sbin/nft", "usr/bin/nft"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_loader = fs::read_dir(runtime.join("lib")) + .ok() + .into_iter() + .flatten() + .filter_map(Result::ok) + .any(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("ld-musl-") && name.ends_with(".so.1")) + }); + if has_ip && has_nft && has_loader { + Ok(()) + } else { + Err(format!( + "trusted supervisor helper runtime '{}' is incomplete", + runtime.display() + )) + } +} + fn ensure_umoci_binary(rootfs: &Path) -> Result<(), String> { let path = rootfs.join(SANDBOX_UMOCI_PATH.trim_start_matches('/')); if UMOCI.is_empty() { @@ -944,10 +1227,52 @@ fn remove_rootfs_path(rootfs: &Path, relative: &str) -> Result<(), String> { #[cfg(test)] mod tests { use super::*; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt as _; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; + #[cfg(target_os = "linux")] + #[test] + fn host_supervisor_cache_rejects_wrong_content_and_installs_atomically() { + let directory = tempfile::tempdir().expect("cache directory"); + let destination = directory.path().join("openshell-sandbox"); + fs::write(&destination, b"stale executable").expect("write stale cache"); + fs::set_permissions(&destination, fs::Permissions::from_mode(0o755)) + .expect("make stale cache executable"); + let expected = sha256_bytes(b"trusted supervisor"); + + assert!(validate_host_supervisor_digest(&destination, expected).is_err()); + install_host_supervisor_atomically(&destination, b"trusted supervisor") + .expect("atomically replace cache"); + validate_host_supervisor_digest(&destination, expected).expect("validate installed cache"); + assert_eq!(fs::read(&destination).unwrap(), b"trusted supervisor"); + assert!(fs::read_dir(directory.path()).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .contains(".tmp-") + })); + } + + #[test] + fn guest_init_gets_driver_owned_ip_launcher_when_image_omits_iproute2() { + let rootfs = tempfile::tempdir().expect("create rootfs"); + ensure_guest_init_ip(rootfs.path()).expect("install guest ip launcher"); + + let launcher = rootfs.path().join("usr/sbin/ip"); + let contents = fs::read_to_string(&launcher).expect("read guest ip launcher"); + assert!(contents.contains("/opt/openshell/bin/openshell-runtime")); + assert!(contents.contains("ld-musl-")); + #[cfg(unix)] + assert_eq!( + fs::metadata(launcher).unwrap().permissions().mode() & 0o777, + 0o755 + ); + } + #[test] fn prepare_sandbox_rootfs_rewrites_guest_layout() { let dir = unique_temp_dir(); @@ -959,10 +1284,10 @@ mod tests { write_fake_runtime_binaries(&rootfs); fs::write( rootfs.join("etc/passwd"), - "root:x:0:0:root:/root:/bin/bash\n", + "root:x:0:0:root:/root:/bin/bash\nsandbox:x:998:997:Sandbox:/sandbox:/bin/sh\n", ) .expect("write passwd"); - fs::write(rootfs.join("etc/group"), "root:x:0:\n").expect("write group"); + fs::write(rootfs.join("etc/group"), "root:x:0:\nsandbox:x:997:\n").expect("write group"); fs::write(rootfs.join("etc/hosts"), "127.0.0.1 localhost\n").expect("write hosts"); fs::create_dir_all(rootfs.join("bin")).expect("create bin"); fs::create_dir_all(rootfs.join("sbin")).expect("create sbin"); @@ -979,6 +1304,16 @@ mod tests { assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file()); assert!(rootfs.join("opt/openshell/bin/umoci").is_file()); + validate_supervisor_runtime(&rootfs).expect("trusted helper runtime remains complete"); + let init_script = fs::read_to_string(rootfs.join("srv/openshell-vm-sandbox-init.sh")) + .expect("read guest init"); + assert!( + init_script.contains("--mode=boundary --boundary-config /etc/openshell/boundary.json") + ); + assert!(!init_script.contains("--topology-backend-name=in-pod")); + assert!(!init_script.contains("@ISOLATION_INTERFACE_VERSION@")); + assert!(!init_script.contains("8.8.8.8")); + assert!(!init_script.contains("VM_NET_")); assert!(rootfs.join("sandbox").is_dir()); assert!(rootfs.join("image-cache").is_dir()); assert!(rootfs.join("lower").is_dir()); @@ -990,18 +1325,14 @@ mod tests { .next() .is_none() ); - assert!( - fs::read_to_string(rootfs.join("etc/passwd")) - .expect("read passwd") - .contains(&format!( - "sandbox:x:{uid}:{uid}:OpenShell Sandbox:/sandbox:/bin/bash" - )) - ); - assert!( - fs::read_to_string(rootfs.join("etc/group")) - .expect("read group") - .contains(&format!("sandbox:x:{uid}:")) - ); + let passwd = fs::read_to_string(rootfs.join("etc/passwd")).expect("read passwd"); + assert!(passwd.contains(&format!( + "sandbox:x:{uid}:{uid}:OpenShell Sandbox:/sandbox:/bin/bash" + ))); + assert!(!passwd.contains("sandbox:x:998:997:")); + let group = fs::read_to_string(rootfs.join("etc/group")).expect("read group"); + assert!(group.contains(&format!("sandbox:x:{uid}:"))); + assert!(!group.contains("sandbox:x:997:")); assert_eq!( fs::read_to_string(rootfs.join("etc/hosts")).expect("read hosts"), "127.0.0.1 localhost\n" @@ -1010,6 +1341,61 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn supervisor_runtime_archive_materializes_below_the_trusted_path() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + let untrusted_runtime = rootfs.join("opt/openshell/bin/openshell-runtime"); + fs::create_dir_all(untrusted_runtime.join("usr/sbin")).expect("create untrusted runtime"); + fs::write(untrusted_runtime.join("usr/sbin/ip"), b"untrusted") + .expect("write untrusted helper"); + fs::write(untrusted_runtime.join("untrusted-extra"), b"untrusted") + .expect("write untrusted extra file"); + let mut tar_bytes = Vec::new(); + { + let mut archive = tar::Builder::new(&mut tar_bytes); + for (path, bytes, mode) in [ + ("openshell-runtime/usr/sbin/ip", b"ip".as_slice(), 0o755), + ("openshell-runtime/usr/sbin/nft", b"nft".as_slice(), 0o755), + ( + "openshell-runtime/lib/ld-musl-test.so.1", + b"loader".as_slice(), + 0o755, + ), + ] { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(mode); + header.set_entry_type(tar::EntryType::Regular); + header.set_cksum(); + archive + .append_data(&mut header, path, bytes) + .expect("append runtime entry"); + } + archive.finish().expect("finish runtime archive"); + } + let compressed = zstd::encode_all(Cursor::new(tar_bytes), 1).expect("compress runtime"); + + install_supervisor_runtime_archive(&rootfs, &compressed).expect("install runtime"); + validate_supervisor_runtime(&rootfs).expect("validate runtime"); + assert!( + rootfs + .join("opt/openshell/bin/openshell-runtime/usr/sbin/nft") + .is_file() + ); + assert_eq!( + fs::read(rootfs.join("opt/openshell/bin/openshell-runtime/usr/sbin/ip")) + .expect("read installed helper"), + b"ip" + ); + assert!( + !rootfs + .join("opt/openshell/bin/openshell-runtime/untrusted-extra") + .exists(), + "embedded runtime replacement must discard bootstrap-image helpers" + ); + } + #[test] fn prepare_sandbox_rootfs_preserves_image_workdir_contents_in_rootfs() { let dir = unique_temp_dir(); @@ -1230,6 +1616,13 @@ mod tests { } fn write_fake_runtime_binaries(rootfs: &Path) { + let helper_runtime = rootfs.join("opt/openshell/bin/openshell-runtime"); + fs::create_dir_all(helper_runtime.join("usr/sbin")).expect("create helper bin directory"); + fs::create_dir_all(helper_runtime.join("lib")).expect("create helper lib directory"); + fs::write(helper_runtime.join("usr/sbin/ip"), b"ip").expect("write ip helper"); + fs::write(helper_runtime.join("usr/sbin/nft"), b"nft").expect("write nft helper"); + fs::write(helper_runtime.join("lib/ld-musl-test.so.1"), b"loader") + .expect("write helper loader"); fs::write( rootfs.join("opt/openshell/bin/openshell-sandbox"), b"sandbox", diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index f6020af829..50014b8f89 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -5,10 +5,10 @@ use std::ffi::CString; use std::path::{Path, PathBuf}; -use std::process::{Child as StdChild, Command as StdCommand, Stdio}; +use std::process::{Command as StdCommand, Stdio}; use std::ptr; use std::sync::atomic::{AtomicI32, Ordering}; -use std::time::{Duration, Instant}; +use std::time::Duration; use crate::{embedded_runtime, ffi, nft_ruleset, procguard, rootfs}; @@ -19,31 +19,18 @@ const KRUN_INIT_PID1_ENV: &str = "KRUN_INIT_PID1=1"; /// Used by the SIGTERM/SIGINT handler to forward signals to the VM. static CHILD_PID: AtomicI32 = AtomicI32::new(0); -/// PID of the helper process (gvproxy for libkrun; zero for QEMU). -/// Zero when not running. Used by the SIGTERM/SIGINT handler and -/// procguard cleanup callback to ensure the helper doesn't outlive the -/// launcher (especially on macOS where `PR_SET_PDEATHSIG` is absent). -static GVPROXY_PID: AtomicI32 = AtomicI32::new(0); - #[derive(Debug, Clone, PartialEq, Eq)] pub enum VmBackend { Libkrun, Qemu, } -// virtio-net feature bits (see Linux `include/uapi/linux/virtio_net.h`). -const NET_FEATURE_CSUM: u32 = 1 << 0; -const NET_FEATURE_GUEST_CSUM: u32 = 1 << 1; -const NET_FEATURE_GUEST_TSO4: u32 = 1 << 7; -const NET_FEATURE_GUEST_UFO: u32 = 1 << 10; -const NET_FEATURE_HOST_TSO4: u32 = 1 << 11; -const NET_FEATURE_HOST_UFO: u32 = 1 << 14; -const COMPAT_NET_FEATURES: u32 = NET_FEATURE_CSUM - | NET_FEATURE_GUEST_CSUM - | NET_FEATURE_GUEST_TSO4 - | NET_FEATURE_GUEST_UFO - | NET_FEATURE_HOST_TSO4 - | NET_FEATURE_HOST_UFO; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VsockPortMap { + pub guest_port: u32, + pub host_socket: PathBuf, + pub host_initiated: bool, +} pub struct VmLaunchConfig { pub root_disk: PathBuf, @@ -66,6 +53,7 @@ pub struct VmLaunchConfig { pub vsock_cid: Option, pub guest_mac: Option, pub gateway_port: Option, + pub vsock_port_map: Option, } pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { @@ -80,25 +68,9 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { .gpu_bdf .as_deref() .ok_or("gpu_bdf is required for QEMU backend")?; - let tap_device = config - .tap_device - .as_deref() - .ok_or("tap_device is required for QEMU backend")?; - let guest_mac = config - .guest_mac - .as_deref() - .ok_or("guest_mac is required for QEMU backend")?; let vsock_cid = config .vsock_cid .ok_or("vsock_cid is required for QEMU backend")?; - let _guest_ip = config - .guest_ip - .as_deref() - .ok_or("guest_ip is required for QEMU backend")?; - let host_ip = config - .host_ip - .as_deref() - .ok_or("host_ip is required for QEMU backend")?; if !config.root_disk.is_file() { return Err(format!( @@ -125,13 +97,9 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { #[cfg(target_os = "linux")] check_kvm_access()?; - let guest_env = qemu_guest_env_vars(config, host_dns_server()); + let guest_env = qemu_guest_env_vars(config); write_guest_env_file(&config.overlay_disk, &guest_env)?; - let gw_port = config.gateway_port.unwrap_or(0); - setup_tap_networking(tap_device, host_ip, gw_port)?; - let mut tap_guard = TapGuard::new(tap_device.to_string(), host_ip.to_string(), gw_port); - let vmlinux = if let Some(kernel_image) = &config.kernel_image { kernel_image.clone() } else { @@ -160,16 +128,6 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { .arg("-append") .arg(&kernel_cmdline) .args(qemu_disk_args(config)) - .arg("-netdev") - .arg(format!( - "tap,id=net0,ifname={tap_device},script=no,downscript=no" - )) - .arg("-device") - .arg("pcie-root-port,id=net_root,slot=3") - .arg("-device") - .arg(format!( - "virtio-net-pci-non-transitional,netdev=net0,mac={guest_mac},bus=net_root" - )) .arg("-device") .arg("pcie-root-port,id=vsock_root,slot=1") .arg("-device") @@ -211,8 +169,6 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> { .map_err(|e| format!("failed to wait for QEMU: {e}"))?; CHILD_PID.store(0, Ordering::Relaxed); - teardown_tap_networking(tap_device, host_ip, gw_port); - tap_guard.disarm(); if status.success() { Ok(()) @@ -271,20 +227,8 @@ fn write_guest_env_file(overlay_disk: &Path, env_vars: &[String]) -> Result<(), ) } -fn qemu_guest_env_vars(config: &VmLaunchConfig, dns_server: Option) -> Vec { +fn qemu_guest_env_vars(config: &VmLaunchConfig) -> Vec { let mut env_vars = config.env.clone(); - - if let Some(ip) = &config.guest_ip - && let Some(host_ip) = &config.host_ip - { - env_vars.push(format!("VM_NET_IP={ip}")); - env_vars.push(format!("VM_NET_GW={host_ip}")); - } - - if let Some(dns) = dns_server { - env_vars.push(format!("VM_NET_DNS={dns}")); - } - if config.gpu_bdf.is_some() { env_vars.push("GPU_ENABLED=true".to_string()); } @@ -312,12 +256,6 @@ fn build_kernel_cmdline(config: &VmLaunchConfig) -> String { format!("init={}", config.exec_path), ]; - if let Some(ip) = &config.guest_ip - && let Some(host_ip) = &config.host_ip - { - parts.push(format!("ip={ip}::{host_ip}:255.255.255.252:sandbox::off")); - } - if config.gpu_bdf.is_some() { parts.push("firmware_class.path=/lib/firmware".to_string()); } @@ -325,29 +263,6 @@ fn build_kernel_cmdline(config: &VmLaunchConfig) -> String { parts.join(" ") } -fn host_dns_server() -> Option { - // Prefer systemd-resolved upstream config (skips the 127.0.0.53 - // stub listener which is unreachable from inside QEMU/TAP guests). - for path in &["/run/systemd/resolve/resolv.conf", "/etc/resolv.conf"] { - let Ok(resolv) = std::fs::read_to_string(path) else { - continue; - }; - for line in resolv.lines() { - let line = line.trim(); - if let Some(server) = line.strip_prefix("nameserver") { - let server = server.trim(); - if server == "127.0.0.53" || server.starts_with("127.") { - continue; - } - if !server.is_empty() { - return Some(server.to_string()); - } - } - } - } - None -} - /// Remove leftover `vmtap-*` interfaces from previous driver runs. /// /// Called once at driver startup for interfaces that were not torn down @@ -400,81 +315,6 @@ fn read_tap_host_ip(device: &str) -> Option { None } -fn setup_tap_networking(tap_device: &str, host_ip: &str, gateway_port: u16) -> Result<(), String> { - run_cmd("ip", &["tuntap", "add", "dev", tap_device, "mode", "tap"])?; - run_cmd( - "ip", - &["addr", "add", &format!("{host_ip}/30"), "dev", tap_device], - )?; - run_cmd("ip", &["link", "set", tap_device, "up"])?; - - // Deprioritize routes through down interfaces so a stale vmtap-* - // that somehow survives cleanup cannot shadow the active one. - let _ = std::fs::write( - format!("/proc/sys/net/ipv4/conf/{tap_device}/ignore_routes_with_linkdown"), - "1", - ); - - enable_ip_forwarding()?; - - let subnet = tap_subnet_from_host_ip(host_ip); - let table_name = nft_ruleset::teardown_table_name(tap_device); - - // Delete any stale nftables table from a previous driver run. - let _ = run_cmd("nft", &["delete", "table", "ip", &table_name]); - - // Clean up legacy iptables rules from older driver versions. - let _ = run_cmd( - "iptables", - &[ - "-t", - "nat", - "-D", - "POSTROUTING", - "-s", - &subnet, - "-j", - "MASQUERADE", - ], - ); - let _ = run_cmd( - "iptables", - &["-D", "FORWARD", "-i", tap_device, "-j", "ACCEPT"], - ); - let _ = run_cmd( - "iptables", - &[ - "-D", - "FORWARD", - "-o", - tap_device, - "-m", - "state", - "--state", - "RELATED,ESTABLISHED", - "-j", - "ACCEPT", - ], - ); - let port_str = gateway_port.to_string(); - let _ = run_cmd( - "iptables", - &[ - "-D", "INPUT", "-i", tap_device, "-p", "tcp", "--dport", &port_str, "-j", "ACCEPT", - ], - ); - let _ = run_cmd( - "iptables", - &["-D", "INPUT", "-i", tap_device, "-j", "ACCEPT"], - ); - - // Load nftables ruleset atomically. - let ruleset = nft_ruleset::generate_tap_ruleset(tap_device, &subnet, gateway_port); - run_nft_stdin(&ruleset)?; - - Ok(()) -} - fn teardown_tap_networking(tap_device: &str, host_ip: &str, gateway_port: u16) { // Delete the entire nftables table — single atomic operation. let table_name = nft_ruleset::teardown_table_name(tap_device); @@ -543,11 +383,6 @@ fn tap_subnet_from_host_ip(host_ip: &str) -> String { ) } -fn enable_ip_forwarding() -> Result<(), String> { - std::fs::write("/proc/sys/net/ipv4/ip_forward", "1") - .map_err(|e| format!("enable ip_forward: {e}")) -} - fn run_cmd(cmd: &str, args: &[&str]) -> Result<(), String> { let output = StdCommand::new(cmd) .args(args) @@ -564,87 +399,16 @@ fn run_cmd(cmd: &str, args: &[&str]) -> Result<(), String> { } } -fn run_nft_stdin(ruleset: &str) -> Result<(), String> { - use std::io::Write; - - let mut child = StdCommand::new("nft") - .args(["-f", "-"]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| format!("failed to run nft: {e}"))?; - - if let Some(mut stdin) = child.stdin.take() { - stdin - .write_all(ruleset.as_bytes()) - .map_err(|e| format!("failed to write nft ruleset: {e}"))?; - } - - let output = child - .wait_with_output() - .map_err(|e| format!("failed to wait for nft: {e}"))?; - - if output.status.success() { - Ok(()) - } else { - let stderr = String::from_utf8_lossy(&output.stderr); - Err(format!("nft -f - failed: {stderr}")) - } -} - -/// RAII guard that tears down TAP networking on drop. -struct TapGuard { - tap_device: String, - host_ip: String, - gateway_port: u16, - disarmed: bool, -} - -impl TapGuard { - fn new(tap_device: String, host_ip: String, gateway_port: u16) -> Self { - Self { - tap_device, - host_ip, - gateway_port, - disarmed: false, - } - } - - fn disarm(&mut self) { - self.disarmed = true; - } -} - -impl Drop for TapGuard { - fn drop(&mut self) { - if !self.disarmed { - teardown_tap_networking(&self.tap_device, &self.host_ip, self.gateway_port); - } - } -} - /// Shared procguard cleanup callback for both libkrun and QEMU paths. /// Only async-signal-safe calls: atomic loads and `kill(2)`. fn procguard_kill_children() { - let helper_pid = GVPROXY_PID.load(Ordering::Relaxed); let child_pid = CHILD_PID.load(Ordering::Relaxed); - if helper_pid > 0 { - unsafe { - libc::kill(helper_pid, libc::SIGTERM); - } - } if child_pid > 0 { unsafe { libc::kill(child_pid, libc::SIGTERM); } } std::thread::sleep(Duration::from_millis(200)); - if helper_pid > 0 { - unsafe { - libc::kill(helper_pid, libc::SIGKILL); - } - } if child_pid > 0 { unsafe { libc::kill(child_pid, libc::SIGKILL); @@ -677,13 +441,9 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { return Err(format!("image disk not found: {}", image_disk.display())); } - // Arm procguard first, BEFORE we spawn gvproxy or fork libkrun, so - // that the launcher can't be orphaned during setup. The cleanup - // callback reads the GVPROXY_PID atomic (initially 0 — no-op) and - // the CHILD_PID atomic (the libkrun fork), so it stays correct as - // those slots get populated later in this function. Only ONE arm - // per process: racing two watchers for the same NOTE_EXIT event - // would cause whichever wins to skip the cleanup. + // Arm procguard before forking libkrun so the VM worker cannot outlive + // the launcher. No network helper is started: the only host/guest data + // path is the protected vsock mapping below. if let Err(err) = procguard::die_with_parent_cleanup(procguard_kill_children) { return Err(format!("procguard arm failed: {err}")); } @@ -705,132 +465,12 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { )?; vm.set_workdir(&config.workdir)?; - // Run gvproxy strictly as the guest's virtual NIC / DHCP / router. - // - // After the supervisor-initiated relay migration (#867), the driver - // no longer forwards any host-side ports into the guest — all ingress - // traffic for SSH and exec rides the outbound `ConnectSupervisor` - // gRPC stream the guest opens to the gateway. What gvproxy still - // provides here is the TCP/IP *plane* the guest kernel needs: - // - // * a virtio-net backend attached to libkrun via a Unix - // SOCK_STREAM (Linux) or SOCK_DGRAM (macOS vfkit), which - // surfaces as `eth0` inside the guest; - // * the DHCP server + default router the guest's udhcpc client - // talks to on boot (IPs 192.168.127.1 / .2, defaults for - // gvisor-tap-vsock); - // * the host-facing gateway identity the guest uses for callbacks: - // gvproxy installs a default NAT entry rewriting `192.168.127.254` - // (the subnet's HostIP) to the host's `127.0.0.1`, and serves - // `host.containers.internal` / `host.docker.internal` / - // `host.openshell.internal` in its embedded DNS pointing at that - // same HostIP. The guest init script seeds /etc/hosts with the - // same mapping so the supervisor reaches the host gateway even - // when gvproxy's DNS isn't in resolv.conf. The gateway IP - // (192.168.127.1) is NOT a host-loopback proxy — it only listens - // on its own service ports (DNS:53, DHCP, HTTP API:80). - // - // That network plane is also what the sandbox supervisor's - // per-sandbox netns (veth pair + nftables, see - // `openshell-sandbox/src/sandbox/linux/netns.rs`) branches off of; - // libkrun's built-in TSI socket impersonation would not satisfy - // those kernel-level primitives. - // - // The `-listen` API socket and `-ssh-port` forwarder are both - // deliberately omitted: nothing in the driver enqueues port - // forwards on the API any more, and the host-side SSH listener is - // dead plumbing. - let gvproxy_guard = { - let gvproxy_binary = runtime_dir.join("gvproxy"); - if !gvproxy_binary.is_file() { - return Err(format!( - "missing runtime file: {}", - gvproxy_binary.display() - )); - } - - let sock_base = gvproxy_socket_base(&config.overlay_disk)?; - let net_sock = sock_base.with_extension("v"); - let _ = std::fs::remove_file(&net_sock); - let _ = std::fs::remove_file(sock_base.with_extension("v-krun.sock")); - - let run_dir = config.overlay_disk.parent().unwrap_or(&config.overlay_disk); - let gvproxy_log = run_dir.join("gvproxy.log"); - let gvproxy_log_file = std::fs::File::create(&gvproxy_log) - .map_err(|e| format!("create gvproxy log {}: {e}", gvproxy_log.display()))?; - - #[cfg(target_os = "linux")] - let (gvproxy_net_flag, gvproxy_net_url) = - ("-listen-qemu", format!("unix://{}", net_sock.display())); - #[cfg(target_os = "macos")] - let (gvproxy_net_flag, gvproxy_net_url) = ( - "-listen-vfkit", - format!("unixgram://{}", net_sock.display()), - ); - - // `-ssh-port -1` tells gvproxy to skip its default SSH forward - // (127.0.0.1:2222 → guest:22). We don't use it — all gateway - // ingress rides the supervisor-initiated relay — and leaving - // the default on would bind a host-side TCP listener per - // sandbox, racing concurrent sandboxes for port 2222 and - // surfacing a misleading "sshd is reachable" endpoint. See - // https://github.com/containers/gvisor-tap-vsock `cmd/gvproxy/main.go` - // (`getForwardsMap` returns an empty map when `sshPort == -1`). - let mut gvproxy_cmd = StdCommand::new(&gvproxy_binary); - gvproxy_cmd - .arg(gvproxy_net_flag) - .arg(&gvproxy_net_url) - .arg("-ssh-port") - .arg("-1") - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(gvproxy_log_file); - - // On Linux the kernel will SIGKILL gvproxy the moment this - // launcher dies (or is SIGKILLed). `pre_exec` runs in the child - // between fork and execve, so the PR_SET_PDEATHSIG flag is - // inherited across execve and applies to gvproxy proper. On - // macOS/BSDs there is no equivalent; we fall back to killing - // gvproxy explicitly from the launcher's procguard cleanup - // callback (see `run_vm` above) and SIGTERM handler - // (see `install_signal_forwarding` below). - #[cfg(target_os = "linux")] - { - use nix::sys::signal::Signal; - use std::os::unix::process::CommandExt as _; - unsafe { - gvproxy_cmd.pre_exec(|| { - nix::sys::prctl::set_pdeathsig(Signal::SIGKILL) - .map_err(|err| std::io::Error::other(format!("pdeathsig: {err}"))) - }); - } - } - - let child = gvproxy_cmd - .spawn() - .map_err(|e| format!("failed to start gvproxy {}: {e}", gvproxy_binary.display()))?; - // The procguard cleanup reads GVPROXY_PID atomically. Storing it - // here makes the callback able to SIGTERM gvproxy if the driver - // dies from this moment onward. - GVPROXY_PID.store(child.id().cast_signed(), Ordering::Relaxed); - - wait_for_path(&net_sock, Duration::from_secs(5), "gvproxy data socket")?; - - vm.disable_implicit_vsock()?; - vm.add_vsock(0)?; - - let mac: [u8; 6] = [0x5a, 0x94, 0xef, 0xe4, 0x0c, 0xee]; - - #[cfg(target_os = "linux")] - vm.add_net_unixstream(&net_sock, &mac, COMPAT_NET_FEATURES)?; - #[cfg(target_os = "macos")] - { - const NET_FLAG_VFKIT: u32 = 1 << 0; - vm.add_net_unixgram(&net_sock, &mac, COMPAT_NET_FEATURES, NET_FLAG_VFKIT)?; - } - - Some(GvproxyGuard::new(child)) - }; + vm.disable_implicit_vsock()?; + vm.add_vsock(0)?; + if let Some(port_map) = &config.vsock_port_map { + let _ = std::fs::remove_file(&port_map.host_socket); + vm.add_vsock_port(port_map)?; + } vm.set_console_output(&config.console_output)?; @@ -864,9 +504,6 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { let status = wait_for_child(pid)?; CHILD_PID.store(0, Ordering::Relaxed); - cleanup_gvproxy(gvproxy_guard); - GVPROXY_PID.store(0, Ordering::Relaxed); - if libc::WIFEXITED(status) { match libc::WEXITSTATUS(status) { 0 => Ok(()), @@ -1118,50 +755,18 @@ impl VmContext { ) } - #[cfg(target_os = "macos")] - fn add_net_unixgram( - &self, - socket_path: &Path, - mac: &[u8; 6], - features: u32, - flags: u32, - ) -> Result<(), String> { - let sock_c = path_to_cstring(socket_path)?; - check( - unsafe { - (self.krun.krun_add_net_unixgram)( - self.ctx_id, - sock_c.as_ptr(), - -1, - mac.as_ptr(), - features, - flags, - ) - }, - "krun_add_net_unixgram", - ) - } - - #[allow(dead_code)] // Used on Linux when gvproxy runs in qemu/unixstream mode. - fn add_net_unixstream( - &self, - socket_path: &Path, - mac: &[u8; 6], - features: u32, - ) -> Result<(), String> { - let sock_c = path_to_cstring(socket_path)?; + fn add_vsock_port(&self, port_map: &VsockPortMap) -> Result<(), String> { + let socket_c = path_to_cstring(&port_map.host_socket)?; check( unsafe { - (self.krun.krun_add_net_unixstream)( + (self.krun.krun_add_vsock_port2)( self.ctx_id, - sock_c.as_ptr(), - -1, - mac.as_ptr(), - features, - 0, + port_map.guest_port, + socket_c.as_ptr(), + port_map.host_initiated, ) }, - "krun_add_net_unixstream", + "krun_add_vsock_port2", ) } @@ -1210,109 +815,6 @@ impl Drop for VmContext { } } -struct GvproxyGuard { - child: Option, -} - -impl GvproxyGuard { - fn new(child: StdChild) -> Self { - Self { child: Some(child) } - } - - fn disarm(&mut self) -> Option { - self.child.take() - } -} - -impl Drop for GvproxyGuard { - fn drop(&mut self) { - if let Some(mut child) = self.child.take() { - let _ = child.kill(); - let _ = child.wait(); - } - } -} - -fn wait_for_path(path: &Path, timeout: Duration, label: &str) -> Result<(), String> { - let deadline = Instant::now() + timeout; - let mut interval = Duration::from_millis(5); - while !path.exists() { - if Instant::now() >= deadline { - return Err(format!( - "{label} did not appear within {:.1}s: {}", - timeout.as_secs_f64(), - path.display() - )); - } - std::thread::sleep(interval); - interval = (interval * 2).min(Duration::from_millis(200)); - } - Ok(()) -} - -fn hash_path_id(path: &Path) -> String { - let mut hash: u64 = 0xcbf2_9ce4_8422_2325; - for byte in path.to_string_lossy().as_bytes() { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(0x0100_0000_01b3); - } - format!("{:012x}", hash & 0x0000_ffff_ffff_ffff) -} - -fn secure_socket_base(subdir: &str) -> Result { - let base = std::env::var_os("XDG_RUNTIME_DIR").map_or_else( - || { - let fallback = PathBuf::from("/tmp"); - if fallback.is_dir() { - fallback - } else { - std::env::temp_dir() - } - }, - PathBuf::from, - ); - let dir = base.join(subdir); - - if dir.exists() { - let meta = dir - .symlink_metadata() - .map_err(|e| format!("lstat {}: {e}", dir.display()))?; - if meta.file_type().is_symlink() { - return Err(format!( - "socket directory {} is a symlink; refusing to use it", - dir.display() - )); - } - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt as _; - let uid = unsafe { libc::getuid() }; - if meta.uid() != uid { - return Err(format!( - "socket directory {} is owned by uid {} but we are uid {}", - dir.display(), - meta.uid(), - uid - )); - } - } - } else { - std::fs::create_dir_all(&dir) - .map_err(|e| format!("create socket dir {}: {e}", dir.display()))?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)); - } - } - - Ok(dir) -} - -fn gvproxy_socket_base(overlay_disk: &Path) -> Result { - Ok(secure_socket_base("osd-gv")?.join(hash_path_id(overlay_disk))) -} - fn install_signal_forwarding(pid: i32) { unsafe { libc::signal( @@ -1327,12 +829,7 @@ fn install_signal_forwarding(pid: i32) { CHILD_PID.store(pid, Ordering::Relaxed); } -/// Async-signal-safe handler that forwards SIGTERM to every process we -/// own: the libkrun VM worker and the gvproxy helper. We cannot rely on -/// Rust destructors (`GvproxyGuard::drop`, `ManagedDriverProcess::drop`) -/// running on signal-driven exit, so we explicitly deliver the signal -/// here. The `wait_for_child` loop reaps libkrun and `cleanup_gvproxy` -/// reaps gvproxy before `run_vm` returns. +/// Async-signal-safe handler that forwards SIGTERM to the VM worker. /// /// Only async-signal-safe libc calls are used — `kill(2)` is listed in /// POSIX.1-2017 as async-signal-safe, atomic loads are lock-free on the @@ -1344,13 +841,6 @@ extern "C" fn forward_signal(_sig: libc::c_int) { libc::kill(vm_pid, libc::SIGTERM); } } - let gv_pid = GVPROXY_PID.load(Ordering::Relaxed); - if gv_pid > 0 { - // gvproxy handles SIGTERM cleanly; no need for SIGKILL. - unsafe { - libc::kill(gv_pid, libc::SIGTERM); - } - } } fn wait_for_child(pid: i32) -> Result { @@ -1365,15 +855,6 @@ fn wait_for_child(pid: i32) -> Result { Ok(status) } -fn cleanup_gvproxy(mut guard: Option) { - if let Some(mut guard) = guard.take() - && let Some(mut child) = guard.disarm() - { - let _ = child.kill(); - let _ = child.wait(); - } -} - fn check(ret: i32, func: &'static str) -> Result<(), String> { if ret < 0 { Err(format!("{func} failed with error code {ret}")) @@ -1437,17 +918,16 @@ mod tests { vsock_cid: Some(4), guest_mac: Some("02:00:00:00:00:01".to_string()), gateway_port: Some(8080), + vsock_port_map: None, } } #[test] - fn qemu_guest_env_vars_include_driver_runtime_metadata() { - let env = qemu_guest_env_vars(&qemu_config(), Some("1.1.1.1".to_string())); + fn qemu_guest_env_vars_omit_network_metadata() { + let env = qemu_guest_env_vars(&qemu_config()); assert!(env.contains(&"OPENSHELL_ENDPOINT=http://10.0.128.1:8080".to_string())); - assert!(env.contains(&"VM_NET_IP=10.0.128.2".to_string())); - assert!(env.contains(&"VM_NET_GW=10.0.128.1".to_string())); - assert!(env.contains(&"VM_NET_DNS=1.1.1.1".to_string())); + assert!(!env.iter().any(|value| value.starts_with("VM_NET_"))); assert!(env.contains(&"GPU_ENABLED=true".to_string())); } @@ -1490,13 +970,13 @@ mod tests { } #[test] - fn kernel_cmdline_keeps_guest_init_metadata_out_of_proc_cmdline() { + fn kernel_cmdline_has_no_guest_network_configuration() { let cmdline = build_kernel_cmdline(&qemu_config()); assert!(cmdline.contains("root=/dev/vda")); assert!(cmdline.contains("rootfstype=ext4")); assert!(cmdline.contains(" ro")); - assert!(cmdline.contains("ip=10.0.128.2::10.0.128.1:255.255.255.252:sandbox::off")); + assert!(!cmdline.contains("ip=")); assert!(cmdline.contains("firmware_class.path=/lib/firmware")); assert!(!cmdline.contains("VM_NET_IP=")); assert!(!cmdline.contains("VM_NET_GW=")); @@ -1537,18 +1017,6 @@ mod tests { assert!(args.contains(&"virtio-blk-pci,drive=image".to_string())); } - #[test] - fn gvproxy_socket_base_is_per_sandbox_overlay_path() { - let first = - gvproxy_socket_base(Path::new("/tmp/openshell-vm/sandboxes/first/overlay.ext4")) - .expect("first socket base"); - let second = - gvproxy_socket_base(Path::new("/tmp/openshell-vm/sandboxes/second/overlay.ext4")) - .expect("second socket base"); - - assert_ne!(first, second); - } - #[test] fn tap_subnet_from_host_ip_calculates_slash30_base() { assert_eq!(tap_subnet_from_host_ip("10.0.128.1"), "10.0.128.0/30"); diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 96da2a879f..7b0a92d4b8 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -25,10 +25,11 @@ # What the script does: # 1. When no prebuilt VM driver is supplied, ensures the VM runtime # (libkrun + gvproxy) and bundled supervisor are staged. -# 2. Builds `openshell-gateway`, `openshell-driver-vm`, and the -# `openshell` CLI with the embedded runtime as needed. When CI supplies -# OPENSHELL_GATEWAY_BIN, OPENSHELL_VM_DRIVER_BIN, or OPENSHELL_BIN, the -# matching prebuilt binary is reused instead of rebuilt. +# 2. Builds `openshell-gateway`, `openshell-driver-vm`, the native host +# `openshell-sandbox` control supervisor, and the `openshell` CLI with the +# embedded runtime as needed. When CI supplies OPENSHELL_GATEWAY_BIN, +# OPENSHELL_VM_DRIVER_BIN, OPENSHELL_VM_SUPERVISOR_BIN, or OPENSHELL_BIN, +# the matching prebuilt binary is reused instead of rebuilt. # 3. On macOS, codesigns the VM driver (libkrun needs the # `com.apple.security.hypervisor` entitlement). # 4. Writes a per-run gateway config with `[openshell.drivers.vm]` @@ -88,10 +89,10 @@ if [ -z "${OPENSHELL_VM_DRIVER_BIN:-}" ]; then mise run vm:setup fi - if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then - echo "==> Building bundled VM supervisor (mise run vm:supervisor)" - mise run vm:supervisor - fi + # Always rebuild the guest bundle so an e2e run cannot silently exercise a + # stale boundary binary after supervisor or isolation-interface changes. + echo "==> Building bundled VM supervisor (mise run vm:supervisor)" + mise run vm:supervisor export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR}}" else @@ -116,6 +117,14 @@ if [ -z "${OPENSHELL_VM_DRIVER_BIN:-}" ]; then else echo "==> Using prebuilt openshell-driver-vm at ${DRIVER_BIN}" fi +if [ -z "${OPENSHELL_VM_SUPERVISOR_BIN:-}" ]; then + # The VM driver prefers a native sibling `openshell-sandbox` for control + # mode. Build it explicitly so a stale target/debug binary cannot disagree + # with the freshly embedded guest boundary protocol. + build_packages+=(-p openshell-sandbox) +else + echo "==> Using prebuilt VM host supervisor at ${OPENSHELL_VM_SUPERVISOR_BIN}" +fi if [ -z "${OPENSHELL_BIN:-}" ]; then build_packages+=(-p openshell-cli) else diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 3818dca364..58a4f8f708 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -309,9 +309,9 @@ if [[ -n "${CARGO_BUILD_JOBS:-}" ]]; then CARGO_BUILD_JOBS_ARG=(-j "${CARGO_BUILD_JOBS}") fi -echo "==> Building openshell-gateway and openshell-driver-vm" +echo "==> Building openshell-gateway, openshell-driver-vm, and native control supervisor" cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \ - -p openshell-gateway -p openshell-driver-vm + -p openshell-gateway -p openshell-driver-vm -p openshell-sandbox if [ "$(uname -s)" = "Darwin" ]; then echo "==> Codesigning openshell-driver-vm (Hypervisor entitlement)" diff --git a/tasks/scripts/vm/build-supervisor-bundle.sh b/tasks/scripts/vm/build-supervisor-bundle.sh index 0085c0619d..a31e999603 100755 --- a/tasks/scripts/vm/build-supervisor-bundle.sh +++ b/tasks/scripts/vm/build-supervisor-bundle.sh @@ -60,6 +60,7 @@ esac SUPERVISOR_BIN="${ROOT}/target/${RUST_TARGET}/release/openshell-sandbox" SUPERVISOR_OUTPUT="${OUTPUT_DIR}/openshell-sandbox.zst" +SUPERVISOR_RUNTIME_OUTPUT="${OUTPUT_DIR}/openshell-runtime.tar.zst" echo "==> Building openshell-sandbox supervisor bundle" echo " Guest arch: ${GUEST_ARCH}" @@ -123,6 +124,51 @@ fi zstd -19 -T0 -f "${SUPERVISOR_BIN}" -o "${SUPERVISOR_OUTPUT}" +case "${GUEST_ARCH}" in + aarch64|arm64) DOCKER_ARCH="arm64" ;; + x86_64|amd64) DOCKER_ARCH="amd64" ;; +esac + +echo "==> Building trusted supervisor helper runtime" +STAGED_SUPERVISOR="${ROOT}/deploy/docker/.build/prebuilt-binaries/${DOCKER_ARCH}/openshell-sandbox" +RUNTIME_IMAGE="openshell-vm-helper-runtime:${DOCKER_ARCH}-$$" +mkdir -p "$(dirname "${STAGED_SUPERVISOR}")" +cp "${SUPERVISOR_BIN}" "${STAGED_SUPERVISOR}" + +case "$(uname -m)" in + aarch64|arm64) HOST_DOCKER_ARCH="arm64" ;; + x86_64|amd64) HOST_DOCKER_ARCH="amd64" ;; + *) HOST_DOCKER_ARCH="" ;; +esac + +if [ "${HOST_DOCKER_ARCH}" = "${DOCKER_ARCH}" ]; then + docker build \ + --build-arg "TARGETARCH=${DOCKER_ARCH}" \ + --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ + --tag "${RUNTIME_IMAGE}" \ + "${ROOT}" +else + docker buildx build \ + --load \ + --platform "linux/${DOCKER_ARCH}" \ + --build-arg "TARGETARCH=${DOCKER_ARCH}" \ + --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ + --tag "${RUNTIME_IMAGE}" \ + "${ROOT}" +fi + +RUNTIME_CONTAINER="$(docker create "${RUNTIME_IMAGE}")" +cleanup_runtime_image() { + docker rm -f "${RUNTIME_CONTAINER}" >/dev/null 2>&1 || true + docker image rm "${RUNTIME_IMAGE}" >/dev/null 2>&1 || true +} +trap cleanup_runtime_image EXIT +docker cp "${RUNTIME_CONTAINER}:/openshell-runtime" - \ + | zstd -19 -T0 -f -o "${SUPERVISOR_RUNTIME_OUTPUT}" +cleanup_runtime_image +trap - EXIT + echo "==> Bundled supervisor ready" echo " Binary: $(du -sh "${SUPERVISOR_BIN}" | cut -f1)" echo " Compressed: $(du -sh "${SUPERVISOR_OUTPUT}" | cut -f1)" +echo " Helper runtime: $(du -sh "${SUPERVISOR_RUNTIME_OUTPUT}" | cut -f1)"