From bf5168538e3ed20b4861d57cfdb08e2915b61e8f Mon Sep 17 00:00:00 2001 From: Aaqa Ishtyaq Date: Fri, 21 Aug 2026 13:34:31 +0530 Subject: [PATCH] feat: recreate /dev/userfaultfd inside the jail Cloud Hypervisor OnDemand restore creates the uffd in-process after dropping privileges. Recreate the host character device and chown it to the jail UID so the VMM does not need vm.unprivileged_userfaultfd=1. --- ARCHITECTURE.md | 6 ++++-- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 6 ++++-- src/linux/jail.rs | 54 ++++++++++++++++++++++++++++++++++++++++++++++- src/linux/mod.rs | 3 ++- 6 files changed, 65 insertions(+), 8 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 796ee15..f376796 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -19,7 +19,7 @@ JSON manifest -> manifest validation -> Linux launch workflow or credential details. - `src/linux/jail.rs` stages the root, applies declared bind mounts, resolves allow-listed VFIO character identities, performs `pivot_root`, and creates - only the KVM/TUN/entropy and declared VFIO nodes required by CH. + only the KVM/TUN/entropy/userfaultfd and declared VFIO nodes required by CH. - `src/linux/cgroup.rs` owns cgroup-v2 discovery, controller delegation through `cgroup.subtree_control`, limit writes, and process attachment. - `src/linux/process.rs` owns namespaces, resource limits, descriptor and @@ -54,7 +54,9 @@ Intentional differences: process, logs, and durable PID directly; - the API socket is created by CH inside the jail, not passed as a listener FD or bind mounted from the host; -- Firecracker-specific userfaultfd support is not exposed; and +- Cloud Hypervisor OnDemand restore needs `/dev/userfaultfd` inside the jail + (recreated from the host character identity, owned by the unprivileged VMM). + Firecracker's external uffd-handler is out of scope for this launcher; and - VFIO groups are supplied by the host allocator; this launcher validates the boundary but does not discover devices or decide assignment policy. diff --git a/Cargo.lock b/Cargo.lock index 8b34022..349210a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,7 +112,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "cloud-hypervisor-jailer" -version = "0.1.10" +version = "0.1.11" dependencies = [ "anyhow", "clap", diff --git a/Cargo.toml b/Cargo.toml index e556de7..07832cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cloud-hypervisor-jailer" -version = "0.1.10" +version = "0.1.11" edition = "2024" rust-version = "1.85" description = "Cloud Hypervisor sandbox launcher for Depot" diff --git a/README.md b/README.md index 7c0d8b1..e205bcf 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,9 @@ On Linux, `launch` requires root and then: - mounts only declared non-symlink sources; - joins a pre-created network namespace when requested; - configures the declared cgroup-v2 values and resource limits; -- creates jailed KVM, TUN, and entropy device nodes; +- creates jailed KVM, TUN, entropy, and (when the host has it) userfaultfd + device nodes so Cloud Hypervisor OnDemand restore can create a uffd without + `vm.unprivileged_userfaultfd=1`; - recreates only explicitly declared canonical VFIO control/group character devices, without bind-mounting host `/dev` or changing host device ownership; - creates a PID namespace when requested; @@ -35,7 +37,7 @@ On Linux, `launch` requires root and then: flowchart LR O["Host orchestrator"] -->|"versioned JSON manifest"| V["validate"] V -->|"pure checks"| L["launch as root"] - L --> J["jail\nmount namespace • bind mounts • pivot_root • KVM/TUN/VFIO"] + L --> J["jail\nmount namespace • bind mounts • pivot_root • KVM/TUN/userfaultfd/VFIO"] L --> C["cgroup v2\ncontroller delegation • limits • lease"] L --> P["process\nnetns/PID ns • rlimits • FD/env cleanup • UID/GID"] J --> CH["Cloud Hypervisor\n--seccomp true"] diff --git a/src/linux/jail.rs b/src/linux/jail.rs index 323a124..b483e87 100644 --- a/src/linux/jail.rs +++ b/src/linux/jail.rs @@ -70,6 +70,31 @@ pub(super) fn resolve_devices(manifest: &Manifest) -> Result .collect::>>() } +const USERFAULTFD_PATH: &str = "/dev/userfaultfd"; + +/// Resolve the host `/dev/userfaultfd` identity before pivot, if the node +/// has one. Cloud Hypervisor OnDemand restore tries this device first, then +/// the `userfaultfd(2)` syscall. Recreating the character device inside the +/// jail (owned by the unprivileged VMM) is the narrow grant upstream +/// recommends, instead of `vm.unprivileged_userfaultfd=1` for every process. +pub(super) fn resolve_userfaultfd() -> Result> { + match fs::symlink_metadata(USERFAULTFD_PATH) { + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err).context("stat /dev/userfaultfd"), + Ok(metadata) => { + if metadata.file_type().is_symlink() || !metadata.file_type().is_char_device() { + bail!("/dev/userfaultfd is not a character device"); + } + let device_id = metadata.rdev(); + Ok(Some(ResolvedDevice { + destination: std::path::PathBuf::from("dev/userfaultfd"), + major: libc::major(device_id) as u32, + minor: libc::minor(device_id) as u32, + })) + } + } +} + fn resolve_device(device: &Device) -> Result { let metadata = fs::symlink_metadata(&device.source) .with_context(|| format!("stat device source {}", device.source.display()))?; @@ -112,7 +137,12 @@ pub(super) fn pivot_into_jail(root: &Path) -> Result<()> { syscall_ok(unsafe { libc::rmdir(old_root.as_ptr()) }).context("remove old root") } -pub(super) fn create_device_nodes(uid: u32, gid: u32, devices: &[ResolvedDevice]) -> Result<()> { +pub(super) fn create_device_nodes( + uid: u32, + gid: u32, + devices: &[ResolvedDevice], + userfaultfd: Option<&ResolvedDevice>, +) -> Result<()> { fs::create_dir_all("/dev/net").context("create jailed dev directory")?; if !devices.is_empty() { fs::create_dir_all("/dev/vfio").context("create jailed VFIO directory")?; @@ -123,6 +153,12 @@ pub(super) fn create_device_nodes(uid: u32, gid: u32, devices: &[ResolvedDevice] // Expose only this non-blocking entropy device; guest workloads never // receive the host /dev filesystem. create_character_device(Path::new("/dev/urandom"), 1, 9)?; + if let Some(device) = userfaultfd { + create_character_device(Path::new("/dev/userfaultfd"), device.major, device.minor) + .context("create jailed /dev/userfaultfd")?; + chown_path(Path::new("/dev/userfaultfd"), uid, gid) + .context("chown jailed /dev/userfaultfd")?; + } for device in devices { let destination = Path::new("/").join(&device.destination); create_character_device(&destination, device.major, device.minor) @@ -280,6 +316,7 @@ fn mount_call(source: Option<&Path>, destination: &Path, flags: libc::c_ulong) - #[cfg(test)] mod tests { use super::artifact_mode; + use super::resolve_userfaultfd; #[test] fn mounted_artifact_modes_are_minimally_permissive() { @@ -288,4 +325,19 @@ mod tests { assert_eq!(artifact_mode(true, true), 0o500); assert_eq!(artifact_mode(true, false), 0o700); } + + #[test] + fn resolve_userfaultfd_is_optional_when_the_host_has_no_device() { + match resolve_userfaultfd() { + Ok(None) => {} + Ok(Some(device)) => { + assert_eq!( + device.destination, + std::path::PathBuf::from("dev/userfaultfd") + ); + assert!(device.major > 0 || device.minor > 0); + } + Err(err) => panic!("resolve_userfaultfd: {err}"), + } + } } diff --git a/src/linux/mod.rs b/src/linux/mod.rs index 0d95319..1c83911 100644 --- a/src/linux/mod.rs +++ b/src/linux/mod.rs @@ -21,6 +21,7 @@ pub(crate) fn launch(manifest: &Manifest) -> Result<()> { .transpose() .context("open network namespace")?; let devices = jail::resolve_devices(manifest)?; + let userfaultfd = jail::resolve_userfaultfd()?; jail::prepare_root(manifest)?; jail::enter_mount_namespace()?; @@ -32,7 +33,7 @@ pub(crate) fn launch(manifest: &Manifest) -> Result<()> { // effective capabilities needed for pivot_root and device setup. process::drop_capability_bounding_set()?; jail::pivot_into_jail(&manifest.root)?; - jail::create_device_nodes(manifest.uid, manifest.gid, &devices)?; + jail::create_device_nodes(manifest.uid, manifest.gid, &devices, userfaultfd.as_ref())?; if let Some(netns) = netns { process::join_network_namespace(netns.as_raw_fd())?; }