From 9cdab5fd884f679f4ac3e25b3032da4dadf62056 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 31 Aug 2026 15:22:57 +0200 Subject: [PATCH 01/11] fix(supervisor-process): avoid shutdown exit report hang Signed-off-by: Evan Lezar --- .../openshell-supervisor-process/src/run.rs | 60 +++++++++++++++---- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index b2820d9588..8c47e789ba 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -453,24 +453,34 @@ pub async fn run_process( .build() ); - if let Some(tx) = sidecar_exit_tx.as_ref() { - report_sidecar_main_process_exit(tx, &main_instance_id, rendered_code).await?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code).await; - info!(instance_id = %main_instance_id, "main-process exit acknowledged"); + if outcome.should_report_main_process_exit() { + if let Some(tx) = sidecar_exit_tx.as_ref() { + report_sidecar_main_process_exit(tx, &main_instance_id, rendered_code).await?; + } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { + report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code) + .await; + info!(instance_id = %main_instance_id, "main-process exit acknowledged"); + } + } else { + info!( + instance_id = %main_instance_id, + "skipping main-process exit report during supervisor shutdown" + ); } main_session.mark_terminal_reported(); - if drain_terminal && terminal_delivery_pending { + if outcome.should_report_main_process_exit() && drain_terminal && terminal_delivery_pending { // The peer's SSH channel-close confirms that the terminal frames sent // above traversed russh and the relay. Detached commands have no active // attachment and never enter this wait. main_session.wait_for_terminal_attachments().await; } - if let Some(tx) = sidecar_exit_tx.as_ref() { - finalize_sidecar_main_process_exit(tx, &main_instance_id).await?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - finalize_main_process_exit_until_ack(endpoint, id, &main_instance_id).await; - info!(instance_id = %main_instance_id, "main-process terminal delivery finalized"); + if outcome.should_report_main_process_exit() { + if let Some(tx) = sidecar_exit_tx.as_ref() { + finalize_sidecar_main_process_exit(tx, &main_instance_id).await?; + } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { + finalize_main_process_exit_until_ack(endpoint, id, &main_instance_id).await; + info!(instance_id = %main_instance_id, "main-process terminal delivery finalized"); + } } supervisor_terminating.store(true, Ordering::Release); @@ -572,6 +582,16 @@ enum ProcessWaitOutcome { }, } +impl ProcessWaitOutcome { + /// A gateway acknowledgement is required for ordinary canonical-process + /// completion, but cannot be awaited after the supervisor itself has been + /// asked to terminate. At that point the gateway may already be shutting + /// down and no longer able to acknowledge the report. + fn should_report_main_process_exit(&self) -> bool { + !matches!(self, Self::ShutdownSignal { .. }) + } +} + async fn wait_for_process_exit_or_shutdown( handle: &mut ProcessHandle, timeout_secs: u64, @@ -789,4 +809,22 @@ mod tests { assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); } + + #[cfg(unix)] + #[test] + fn supervisor_shutdown_exit_skips_gateway_acknowledgement() { + use std::os::unix::process::ExitStatusExt; + + let status = ProcessStatus::from(std::process::ExitStatus::from_raw(libc::SIGTERM)); + + assert!(ProcessWaitOutcome::Exited(status).should_report_main_process_exit()); + assert!(ProcessWaitOutcome::TimedOut.should_report_main_process_exit()); + assert!( + !ProcessWaitOutcome::ShutdownSignal { + signal: "SIGTERM", + status, + } + .should_report_main_process_exit() + ); + } } From 13aaef6c8fc7da97f9f02088bb3dc46f07914e9f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 31 Aug 2026 15:43:07 +0200 Subject: [PATCH 02/11] fix(podman): retain workload signal capability Signed-off-by: Evan Lezar --- crates/openshell-driver-podman/src/container.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index abb9d69dd2..a81ee13e1d 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1114,8 +1114,6 @@ pub fn build_container_spec_for_image( "DAC_OVERRIDE".into(), // Not needed: the supervisor does not create setuid/setgid executables. "FSETID".into(), - // Not needed: the supervisor does not send signals to arbitrary processes. - "KILL".into(), // Not needed: the supervisor does not bind privileged ports (<1024). "NET_BIND_SERVICE".into(), // Not in Podman's default set but explicitly denied in case the image @@ -1146,6 +1144,9 @@ pub fn build_container_spec_for_image( // Child setup clears the capability bounding set before exec, which // requires CAP_SETPCAP in the supervisor until drop_privileges(). "SETPCAP".into(), + // Forwarding shutdown signals to the canonical workload process + // group after it drops to the sandbox UID requires CAP_KILL. + "KILL".into(), ], // SETUID, SETGID, SETPCAP, CHOWN, and FOWNER are intentionally kept from // Podman's default set and not dropped: @@ -1885,6 +1886,7 @@ mod tests { "missing DAC_READ_SEARCH" ); assert!(added.contains(&"SETPCAP"), "missing SETPCAP"); + assert!(added.contains(&"KILL"), "missing KILL"); // SETUID and SETGID are NOT in cap_add — they remain available from the // default bounding set because we no longer use cap_drop:ALL. Verify they @@ -1916,6 +1918,10 @@ mod tests { !dropped.contains(&"SETPCAP"), "SETPCAP must not be dropped (needed for child bounding-set clear)" ); + assert!( + !dropped.contains(&"KILL"), + "KILL must not be dropped (needed to signal the sandbox workload on shutdown)" + ); assert!( !dropped.contains(&"ALL"), "must not use cap_drop:ALL in rootless Podman" From 4f1d8c7f747082e804e63c9748c85978a101b395 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 31 Aug 2026 16:20:10 +0200 Subject: [PATCH 03/11] docs(agents): diagnose rootless Podman shutdown timeout Signed-off-by: Evan Lezar --- .agents/skills/debug-openshell-cluster/SKILL.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 90b9a38409..52afac8649 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -262,6 +262,10 @@ When `userns` is configured (e.g. `userns = "auto"` or `userns = "keep-id"`): rootful Podman uses absolute host IDs (e.g. `uidmap = ["0:1000:1", "1:100000:65536"]`). - `nomap` (without hyphen) is accepted as input but canonicalized to `no-map` for Podman's API. +- A workload remains in `stopping` until Podman resorts to `SIGKILL`: inspect + supervisor logs for `failed to signal entrypoint process group`. The + supervisor must retain `CAP_KILL` so its root process can forward `SIGTERM` + to a workload that runs as the sandbox user. ### Step 6: Check Kubernetes Helm Gateways From 0d2c8cc5b443bff3b6131472a4d3831a28279c4e Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 31 Aug 2026 12:07:48 +0200 Subject: [PATCH 04/11] test(e2e): remove unused capbset probe Signed-off-by: Evan Lezar --- .github/actions/setup-e2e-podman/action.yml | 14 --- e2e/support/capbset-probe.c | 113 -------------------- 2 files changed, 127 deletions(-) delete mode 100644 e2e/support/capbset-probe.c diff --git a/.github/actions/setup-e2e-podman/action.yml b/.github/actions/setup-e2e-podman/action.yml index 47f57f5be8..33a54518da 100644 --- a/.github/actions/setup-e2e-podman/action.yml +++ b/.github/actions/setup-e2e-podman/action.yml @@ -110,17 +110,3 @@ runs: INPUTS_PODMAN_MAJOR: ${{ inputs.podman-major }} INPUTS_PODMAN_PACKAGE_VERSION: ${{ inputs.podman-package-version }} INPUTS_CONMON_PACKAGE_VERSION: ${{ inputs.conmon-package-version }} - - - name: Probe rootless capability bounding set - shell: bash - run: | - set -euo pipefail - probe="$RUNNER_TEMP/openshell-capbset-probe" - cc -static -O2 -Wall -Wextra -Werror \ - e2e/support/capbset-probe.c \ - -o "$probe" - podman run --rm \ - --cap-add=SETPCAP \ - --volume "$probe:/openshell-capbset-probe:ro" \ - docker.io/library/alpine:3.22 \ - /openshell-capbset-probe diff --git a/e2e/support/capbset-probe.c b/e2e/support/capbset-probe.c deleted file mode 100644 index bc93d766db..0000000000 --- a/e2e/support/capbset-probe.c +++ /dev/null @@ -1,113 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Verify the capability-bounding-set condition behind issue #2069 from -// inside a rootless Podman container. - -#include -#include -#include -#include -#include -#include - -static unsigned long long status_capability(const char *field) { - FILE *status = fopen("/proc/self/status", "r"); - if (status == NULL) { - perror("fopen(/proc/self/status)"); - exit(EXIT_FAILURE); - } - - char line[256]; - unsigned long long value = 0; - int found = 0; - while (fgets(line, sizeof(line), status) != NULL) { - char name[32]; - unsigned long long candidate; - if (sscanf(line, "%31[^:]:%llx", name, &candidate) == 2 && - strcmp(name, field) == 0) { - value = candidate; - found = 1; - break; - } - } - fclose(status); - - if (!found) { - fprintf(stderr, "missing %s in /proc/self/status\n", field); - exit(EXIT_FAILURE); - } - return value; -} - -static void print_apparmor_profile(void) { - FILE *profile = fopen("/proc/self/attr/current", "r"); - if (profile == NULL) { - perror("fopen(/proc/self/attr/current)"); - return; - } - - char line[256]; - if (fgets(line, sizeof(line), profile) != NULL) { - printf("apparmor_profile=%s", line); - if (strchr(line, '\n') == NULL) { - putchar('\n'); - } - } - fclose(profile); -} - -int main(int argc, char **argv) { - if (argc != 1) { - fprintf(stderr, "usage: %s\n", argv[0]); - return EXIT_FAILURE; - } - - const unsigned long long setpcap_mask = 1ULL << CAP_SETPCAP; - const unsigned long long cap_bnd_before = status_capability("CapBnd"); - const unsigned long long cap_eff_before = status_capability("CapEff"); - const int setpcap_before = prctl(PR_CAPBSET_READ, CAP_SETPCAP, 0, 0, 0); - if (setpcap_before == -1) { - perror("prctl(PR_CAPBSET_READ) before drop"); - return EXIT_FAILURE; - } - - print_apparmor_profile(); - printf("cap_bnd_before=%016llx\n", cap_bnd_before); - printf("cap_eff_before=%016llx\n", cap_eff_before); - printf("setpcap_bounding_before=%d\n", setpcap_before); - - if (cap_bnd_before == 0 || (cap_bnd_before & setpcap_mask) == 0 || - (cap_eff_before & setpcap_mask) == 0 || setpcap_before != 1) { - fprintf(stderr, "CAP_SETPCAP must be effective and present in a non-empty bounding set\n"); - return EXIT_FAILURE; - } - - errno = 0; - const int drop_result = prctl(PR_CAPBSET_DROP, CAP_SETPCAP, 0, 0, 0); - const int drop_errno = errno; - const unsigned long long cap_bnd_after = status_capability("CapBnd"); - const int setpcap_after = prctl(PR_CAPBSET_READ, CAP_SETPCAP, 0, 0, 0); - - printf("drop_result=%d\n", drop_result); - printf("drop_errno=%d (%s)\n", drop_errno, strerror(drop_errno)); - printf("cap_bnd_after=%016llx\n", cap_bnd_after); - printf("setpcap_bounding_after=%d\n", setpcap_after); - - if (drop_result == 0) { - if (setpcap_after != 0 || (cap_bnd_after & setpcap_mask) != 0) { - fprintf(stderr, "CAP_SETPCAP remained in the bounding set after a successful drop\n"); - return EXIT_FAILURE; - } - } else if (drop_errno == EPERM) { - if (setpcap_after != 1 || (cap_bnd_after & setpcap_mask) == 0) { - fprintf(stderr, "CAP_SETPCAP changed in the bounding set after EPERM\n"); - return EXIT_FAILURE; - } - } else { - fprintf(stderr, "unexpected PR_CAPBSET_DROP result\n"); - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; -} From 5929309167b7ae87f476a3755764405fd0e739e2 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 31 Aug 2026 12:07:55 +0200 Subject: [PATCH 05/11] test(guest): add Ubuntu 26 rootless Podman guest Signed-off-by: Evan Lezar --- nix/test-guest/README.md | 44 ++++----- nix/test-guest/cache.sh | 4 +- .../configuration/podman-rootless.yml | 89 +++++++++++++++++++ nix/test-guest/default.nix | 4 +- .../distros/{ubuntu.nix => ubuntu-24-04.nix} | 0 nix/test-guest/distros/ubuntu-26-04.nix | 25 ++++++ nix/test-guest/run.sh | 6 +- 7 files changed, 146 insertions(+), 26 deletions(-) create mode 100644 nix/test-guest/configuration/podman-rootless.yml rename nix/test-guest/distros/{ubuntu.nix => ubuntu-24-04.nix} (100%) create mode 100644 nix/test-guest/distros/ubuntu-26-04.nix diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index b608cb19fe..a65baf02a3 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -33,12 +33,14 @@ nix/test-guest/ ├── cache-lib.sh ├── cache-seal.sh ├── distros/ -│ ├── ubuntu.nix +│ ├── ubuntu-24-04.nix +│ ├── ubuntu-26-04.nix │ ├── centos.nix │ ├── fedora.nix │ └── rocky.nix └── configuration/ ├── docker.yml + ├── podman-rootless.yml ├── podman.yml └── selinux.yml ``` @@ -56,12 +58,13 @@ The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-gues ## Supported configurations -| Distro | Docker | Podman | SELinux | Package format | -| --- | --- | --- | --- | --- | -| Ubuntu 24.04 | Yes | Yes | No | `.deb` | -| CentOS Stream 10 | No | Yes | Yes | `.rpm` | -| Fedora 44 | No | Yes | Yes | `.rpm` | -| Rocky Linux 9 | Yes | Yes | Yes | `.rpm` | +| Distro | Docker | Podman | Rootless Podman | SELinux | Package format | +| --- | --- | --- | --- | --- | --- | +| Ubuntu 24.04 | Yes | Yes | No | No | `.deb` | +| Ubuntu 26.04 | Yes | Yes | Yes | No | `.deb` | +| CentOS Stream 10 | No | Yes | No | Yes | `.rpm` | +| Fedora 44 | No | Yes | No | Yes | `.rpm` | +| Rocky Linux 9 | Yes | Yes | No | Yes | `.rpm` | The `snapd` configuration is available for Ubuntu and prepares snapd for local Snap lifecycle experiments. It does not install Docker, because the Snap @@ -70,8 +73,8 @@ interface rather than the host-package Docker configuration. The Ubuntu 24.04 Podman configuration is available for runtime and packaging checks, but its Podman 4 release does not provide the `pasta` rootless network -helper required by OpenShell sandbox callbacks. OpenShell Podman E2E runs use -the Fedora guest, which provides Podman 5 and `pasta`. +helper required by OpenShell sandbox callbacks. Rootless Podman E2E uses the +Ubuntu 26.04 guest with `--with podman-rootless`. List the available distros and configurations: @@ -84,13 +87,13 @@ nix run .#test-guest -- --list Boot a base Ubuntu VM: ```shell -nix run .#test-guest -- --distro ubuntu +nix run .#test-guest -- --distro ubuntu-24-04 ``` Apply the Docker configuration before opening the SSH session: ```shell -nix run .#test-guest -- --distro ubuntu --with docker +nix run .#test-guest -- --distro ubuntu-24-04 --with docker ``` Other combinations use the same interface: @@ -99,13 +102,14 @@ Other combinations use the same interface: nix run .#test-guest -- --distro rocky --with docker nix run .#test-guest -- --distro centos --with podman nix run .#test-guest -- --distro fedora --with podman +nix run .#test-guest -- --distro ubuntu-26-04 --with podman-rootless ``` Configurations are repeatable: ```shell nix run .#test-guest -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with docker \ --with podman ``` @@ -138,7 +142,7 @@ The `test-guest-cache` app ensures a prepared disk exists for one exact distro, ```shell nix run .#test-guest-cache -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with docker ``` @@ -147,7 +151,7 @@ backing cache: ```shell nix run .#test-guest-cache -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with docker \ --repository ghcr.io/nvidia/openshell/test-guest-cache \ --digest sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef @@ -157,7 +161,7 @@ The command never publishes implicitly. Add `--push` after authenticating ORAS t ```shell nix run .#test-guest-cache -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with docker \ --repository ghcr.io/nvidia/openshell/test-guest-cache \ --push @@ -184,7 +188,7 @@ The default cache directory is `${XDG_CACHE_HOME:-$HOME/.cache}/openshell/test-g Cache command options: ```text ---distro NAME Base distro: ubuntu, centos, fedora, or rocky +--distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky --with NAME Apply docker, podman, or selinux; repeatable --repository REF OCI repository without a tag --digest DIGEST Trusted OCI manifest digest required for pulls @@ -209,7 +213,7 @@ Install the package in an Ubuntu VM and run a command: ```shell nix run .#test-guest -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with docker \ --install artifacts/openshell_0.0.0-local_arm64.deb \ -- openshell --version @@ -226,7 +230,7 @@ guest file preserves the source's ordinary permission bits: ```shell nix run .#test-guest -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --copy ./openshell:/usr/local/bin/openshell \ -- openshell --version ``` @@ -241,7 +245,7 @@ gateway. On each failure it prints snapd and gateway journals. ```shell nix run .#test-guest -- \ - --distro ubuntu \ + --distro ubuntu-24-04 \ --with snapd \ --keep \ --copy ./openshell_*.snap:/tmp/openshell.snap \ @@ -259,7 +263,7 @@ The destination must be an absolute guest path. Copied files are installed with ## Runner options ```text ---distro NAME Base distro: ubuntu, centos, fedora, or rocky +--distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky --with NAME Apply docker, podman, or selinux; repeatable --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy a regular file into the guest, preserving its host mode; diff --git a/nix/test-guest/cache.sh b/nix/test-guest/cache.sh index 1103b3c289..35e3ac441c 100644 --- a/nix/test-guest/cache.sh +++ b/nix/test-guest/cache.sh @@ -12,8 +12,8 @@ Usage: nix run .#test-guest-cache -- --distro DISTRO [OPTIONS] Options: - --distro NAME Base distro: ubuntu, centos, fedora, or rocky - --with NAME Apply a configuration; repeatable (docker, podman, selinux) + --distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky + --with NAME Apply a configuration; repeatable (docker, podman, podman-rootless, selinux) --repository REF OCI repository without a tag --digest DIGEST Trusted OCI manifest digest required for pulls --cache-dir PATH Override the local prepared-disk cache directory diff --git a/nix/test-guest/configuration/podman-rootless.yml b/nix/test-guest/configuration/podman-rootless.yml new file mode 100644 index 0000000000..dc99934f74 --- /dev/null +++ b/nix/test-guest/configuration/podman-rootless.yml @@ -0,0 +1,89 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Configure rootless Podman in a disposable test guest. + +- name: Configure rootless Podman + hosts: test_vm + become: true + gather_facts: true + + tasks: + - name: Validate rootless Podman support + ansible.builtin.assert: + that: + - ansible_facts.distribution == "Ubuntu" + - ansible_facts.distribution_version is version("26.04", ">=") + fail_msg: >- + Rootless Podman requires Ubuntu 26.04 or newer, not + {{ ansible_facts.distribution }} {{ ansible_facts.distribution_version }}. + + - name: Refresh Ubuntu package metadata + ansible.builtin.apt: + update_cache: true + + - name: Install Ubuntu rootless Podman + ansible.builtin.apt: + name: + - apparmor + - fuse-overlayfs + - passt + - podman + - uidmap + state: present + + - name: Allow pasta to receive Podman stop signals + ansible.builtin.lineinfile: + path: /etc/apparmor.d/usr.bin.pasta + insertafter: "^ include $" + line: " signal (receive) peer=podman," + state: present + register: pasta_apparmor_profile + + - name: Reload pasta AppArmor profile + ansible.builtin.command: + argv: + - apparmor_parser + - --replace + - /etc/apparmor.d/usr.bin.pasta + when: pasta_apparmor_profile.changed + changed_when: pasta_apparmor_profile.changed + + - name: Enable the rootless Podman API socket + ansible.builtin.systemd_service: + name: podman.socket + scope: user + enabled: true + state: started + become: false + + - name: Verify rootless Podman + ansible.builtin.command: + cmd: podman info + become: false + changed_when: false + + - name: Verify rootless Podman mode + ansible.builtin.command: + argv: + - podman + - info + - --format + - "{% raw %}{{.Host.Security.Rootless}}{% endraw %}" + become: false + changed_when: false + register: podman_rootless + failed_when: podman_rootless.stdout != "true" + + - name: Verify rootless Podman uses pasta + ansible.builtin.command: + argv: + - podman + - info + - --format + - "{% raw %}{{.Host.RootlessNetworkCmd}}{% endraw %}" + become: false + changed_when: false + register: podman_rootless_network + failed_when: podman_rootless_network.stdout != "pasta" diff --git a/nix/test-guest/default.nix b/nix/test-guest/default.nix index 753f266ab4..b2249e1f4e 100644 --- a/nix/test-guest/default.nix +++ b/nix/test-guest/default.nix @@ -18,7 +18,8 @@ let if isAarch64 then "${qemu}/bin/qemu-system-aarch64" else "${qemu}/bin/qemu-system-x86_64"; distros = { - ubuntu = import ./distros/ubuntu.nix { inherit pkgs architecture; }; + ubuntu-24-04 = import ./distros/ubuntu-24-04.nix { inherit pkgs architecture; }; + ubuntu-26-04 = import ./distros/ubuntu-26-04.nix { inherit pkgs architecture; }; centos = import ./distros/centos.nix { inherit pkgs architecture; }; fedora = import ./distros/fedora.nix { inherit pkgs architecture; }; rocky = import ./distros/rocky.nix { inherit pkgs architecture; }; @@ -27,6 +28,7 @@ let configurations = { docker = ./configuration/docker.yml; podman = ./configuration/podman.yml; + podman-rootless = ./configuration/podman-rootless.yml; selinux = ./configuration/selinux.yml; snapd = ./configuration/snapd.yml; }; diff --git a/nix/test-guest/distros/ubuntu.nix b/nix/test-guest/distros/ubuntu-24-04.nix similarity index 100% rename from nix/test-guest/distros/ubuntu.nix rename to nix/test-guest/distros/ubuntu-24-04.nix diff --git a/nix/test-guest/distros/ubuntu-26-04.nix b/nix/test-guest/distros/ubuntu-26-04.nix new file mode 100644 index 0000000000..7f37fec138 --- /dev/null +++ b/nix/test-guest/distros/ubuntu-26-04.nix @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, architecture }: + +let + imageArchitecture = if architecture == "aarch64" then "arm64" else "amd64"; + imageUrl = "https://cloud-images.ubuntu.com/releases/releases/26.04/release/ubuntu-26.04-server-cloudimg-${imageArchitecture}.img"; + imageHash = + if architecture == "aarch64" then + "sha256-PhE/3UHznhNyk3UXO7KueT+H3G20KU5SUf8kdpcXiLo=" + else + "sha256-gZa+nXlYBZy1bGx1yA/fbO6KiIW8FJ6nkdfbHH75MDU="; +in +{ + osId = "ubuntu"; + osVersion = "26.04"; + packageFamily = "deb"; + inherit imageUrl imageHash; + image = pkgs.fetchurl { + name = "ubuntu-26.04-server-cloudimg-${imageArchitecture}.img"; + url = imageUrl; + hash = imageHash; + }; +} diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index 6be58eb235..9e5d19baef 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -12,8 +12,8 @@ Usage: nix run .#test-guest -- --distro DISTRO [OPTIONS] [-- COMMAND...] Options: - --distro NAME Base distro: ubuntu, centos, fedora, or rocky - --with NAME Apply a configuration; repeatable (docker, podman, selinux, snapd) + --distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky + --with NAME Apply a configuration; repeatable (docker, podman, podman-rootless, selinux, snapd) --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy a regular file to an absolute guest path, preserving its host mode; repeatable @@ -49,7 +49,7 @@ preserved_file_mode() { local source_mode if [ "$(uname -s)" = Darwin ]; then - if ! source_mode=$(stat -f '%Lp' "${source_path}"); then + if ! source_mode=$(/usr/bin/stat -f '%Lp' "${source_path}"); then echo "could not determine mode for --copy source: ${source_path}" >&2 return 1 fi From 841f8c5106a30b72f82f5768db4e4dd317695fb5 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 31 Aug 2026 12:08:03 +0200 Subject: [PATCH 06/11] test(e2e): run Podman E2E tests inside test guest Signed-off-by: Evan Lezar --- TESTING.md | 6 + e2e/run.sh | 479 +++++++++++++++++++++++---- e2e/rust/tests/live_policy_update.rs | 26 +- e2e/rust/tests/transparent_tcp.rs | 7 + tasks/test.toml | 4 + 5 files changed, 449 insertions(+), 73 deletions(-) diff --git a/TESTING.md b/TESTING.md index 6c0829060d..94e457d6ba 100644 --- a/TESTING.md +++ b/TESTING.md @@ -175,6 +175,12 @@ Run the Podman-backed Rust CLI e2e suite: mise run e2e:podman ``` +Run the rootless Podman suite in an Ubuntu 26.04 Nix test guest: + +```shell +mise run e2e:podman:rootless +``` + Run the VM-backed Rust CLI e2e suite: ```shell diff --git a/e2e/run.sh b/e2e/run.sh index 0505730f05..bf364f6176 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # Build the current checkout, run its gateway on the host or in a disposable -# Nix test guest, and execute one named host-side E2E suite against that gateway. +# Nix test guest, and execute E2E tests against that gateway. set -Eeuo pipefail @@ -20,19 +20,27 @@ usage() { cat <<'EOF' Usage: e2e/run.sh [--vm DISTRO] [--with CONFIG ...] \ - --gateway-config PATH --suite NAME + --gateway-config PATH [--features FEATURES] [--suite NAME] Options: --vm DISTRO Run the gateway in a Nix test guest --with CONFIG Apply a Nix test-guest configuration; repeatable + --tests-in-vm Prebuild Linux Rust E2E test binaries on the host, + copy them into the Nix test guest, and run them there + --cli-bin PATH Use a prebuilt openshell CLI instead of building it + --gateway-bin PATH Use a prebuilt openshell-gateway instead of building it + --sandbox-bin PATH Use a prebuilt openshell-sandbox instead of building it --gateway-config PATH Fully resolved gateway TOML + --features FEATURES Rust e2e feature set to enable (default: e2e) --suite NAME Rust suite at e2e/rust/tests/NAME.rs -h, --help Show this help Omit --vm and --with to run the gateway on the host. Supplying --with without --vm selects Fedora for the Podman driver and Ubuntu otherwise. Set -OPENSHELL_E2E_KEEP=1 to retain state. +OPENSHELL_CLI_BIN for a default --cli-bin; otherwise --tests-in-vm +cross-builds the guest CLI with cargo-zigbuild. Set OPENSHELL_E2E_KEEP=1 to +retain state. EOF } @@ -92,7 +100,12 @@ catalog_has_entry() { vm= gateway_config= +gateway_bin= +cli_bin= +sandbox_bin= +e2e_features=e2e suite_name= +tests_in_vm=0 with_configurations=() while [ "$#" -gt 0 ]; do @@ -107,11 +120,35 @@ while [ "$#" -gt 0 ]; do with_configurations+=("$2") shift 2 ;; + --tests-in-vm) + tests_in_vm=1 + shift + ;; + --cli-bin) + require_value "$1" "$#" "${2:-}" + cli_bin="$(resolve_file "$2")" || die "--cli-bin does not name a file: $2" + shift 2 + ;; + --gateway-bin) + require_value "$1" "$#" "${2:-}" + gateway_bin="$(resolve_file "$2")" || die "--gateway-bin does not name a file: $2" + shift 2 + ;; + --sandbox-bin) + require_value "$1" "$#" "${2:-}" + sandbox_bin="$(resolve_file "$2")" || die "--sandbox-bin does not name a file: $2" + shift 2 + ;; --gateway-config) require_value "$1" "$#" "${2:-}" gateway_config=$2 shift 2 ;; + --features) + require_value "$1" "$#" "${2:-}" + e2e_features=$2 + shift 2 + ;; --suite) require_value "$1" "$#" "${2:-}" suite_name=$2 @@ -130,26 +167,31 @@ done if [ -z "${gateway_config}" ]; then die "--gateway-config is required" fi -if [ -z "${suite_name}" ]; then - die "--suite is required" -fi if ! command -v python3 >/dev/null 2>&1; then die "python3 is required" fi +if ! command -v mise >/dev/null 2>&1; then + die "mise is required to build OpenShell" +fi gateway_config_source=${gateway_config} if ! gateway_config="$(resolve_file "${gateway_config_source}")"; then die "gateway config does not exist: ${gateway_config_source}" fi -gateway_driver="$(python3 -c ' +gateway_driver="$(mise x -- python3 -c ' import sys, tomllib print(tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"]["compute_drivers"][0]) ' "${gateway_config}")" -if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then - die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}" +if [ -z "${e2e_features}" ]; then + die "--features must not be empty" fi -suite_path="${ROOT}/e2e/rust/tests/${suite_name}.rs" -if [ ! -f "${suite_path}" ]; then - die "unknown suite: ${suite_name}" +if [ -n "${suite_name}" ]; then + if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9_-]*$ ]]; then + die "suite name must contain only lowercase letters, digits, underscores, and hyphens: ${suite_name}" + fi + suite_path="${ROOT}/e2e/rust/tests/${suite_name}.rs" + if [ ! -f "${suite_path}" ]; then + die "unknown suite: ${suite_name}" + fi fi mode=host if [ -n "${vm}" ] || [ "${#with_configurations[@]}" -gt 0 ]; then @@ -158,7 +200,7 @@ if [ -n "${vm}" ] || [ "${#with_configurations[@]}" -gt 0 ]; then if [ "${gateway_driver}" = podman ]; then vm=fedora else - vm=ubuntu + vm=ubuntu-24-04 fi fi fi @@ -171,9 +213,6 @@ if [ "${mode}" = vm ]; then die "invalid VM configuration name: ${configuration}" fi done - if [ "${gateway_driver}" = podman ] && [ "${vm}" = ubuntu ]; then - die "the Ubuntu 24.04 guest lacks the Podman 5 pasta helper required for sandbox callbacks; use --vm fedora --with podman" - fi if ! command -v nix >/dev/null 2>&1; then die "Nix is required for VM mode" fi @@ -191,15 +230,17 @@ if [ "${mode}" = vm ]; then die "unknown VM configuration in the Nix test-guest catalog: ${configuration}" fi done +elif [ "${tests_in_vm}" -eq 1 ]; then + die "--tests-in-vm requires --vm" +fi +if [ -z "${cli_bin}" ] && [ -n "${OPENSHELL_CLI_BIN:-}" ]; then + cli_bin="$(resolve_file "${OPENSHELL_CLI_BIN}")" || + die "OPENSHELL_CLI_BIN does not name a file: ${OPENSHELL_CLI_BIN}" fi - gateway_ready_timeout=${OPENSHELL_E2E_GATEWAY_READY_TIMEOUT:-600} if [[ ! ${gateway_ready_timeout} =~ ^[1-9][0-9]*$ ]]; then die "OPENSHELL_E2E_GATEWAY_READY_TIMEOUT must be a positive integer" fi -if ! command -v mise >/dev/null 2>&1; then - die "mise is required to build OpenShell" -fi if ! command -v openssl >/dev/null 2>&1; then die "OpenSSL is required to generate sandbox JWT keys" fi @@ -230,56 +271,93 @@ target_dir="$(e2e_cargo_target_dir "${ROOT}" mise x -- cargo)" ensure_build_nofile_limit -echo "==> Building native host openshell CLI" -mise x -- cargo build "${cargo_jobs[@]}" -p openshell-cli --bin openshell -host_cli_bin="${target_dir}/debug/openshell" - -echo "==> Preparing ${linux_musl_target} build target" -mise x -- rustup target add "${linux_musl_target}" >/dev/null +if [ "${tests_in_vm}" -eq 1 ]; then + if [ -n "${cli_bin}" ]; then + echo "==> Using Linux guest openshell CLI: ${cli_bin}" + else + echo "==> Building Linux guest openshell CLI (${linux_musl_target})" + mise x -- rustup target add "${linux_musl_target}" >/dev/null + ( + export CXXSTDLIB=c++ + mise x -- cargo zigbuild "${cargo_jobs[@]+"${cargo_jobs[@]}"}" \ + --release \ + --target "${linux_musl_target}" \ + -p openshell-cli \ + --bin openshell + ) + cli_bin="${target_dir}/${linux_musl_target}/release/openshell" + fi +elif [ -n "${cli_bin}" ]; then + echo "==> Using host openshell CLI: ${cli_bin}" +else + echo "==> Building native host openshell CLI" + mise x -- cargo build "${cargo_jobs[@]+"${cargo_jobs[@]}"}" -p openshell-cli --bin openshell + cli_bin="${target_dir}/debug/openshell" +fi -echo "==> Building Linux openshell-sandbox (${linux_musl_target})" -mise x -- cargo zigbuild "${cargo_jobs[@]}" \ - --release \ - --target "${linux_musl_target}" \ - -p openshell-sandbox \ - --bin openshell-sandbox -linux_sandbox_bin="${target_dir}/${linux_musl_target}/release/openshell-sandbox" +if [ -n "${sandbox_bin}" ]; then + echo "==> Using Linux openshell-sandbox: ${sandbox_bin}" + linux_sandbox_bin="${sandbox_bin}" +else + echo "==> Preparing ${linux_musl_target} build target" + mise x -- rustup target add "${linux_musl_target}" >/dev/null + + echo "==> Building Linux openshell-sandbox (${linux_musl_target})" + mise x -- cargo zigbuild "${cargo_jobs[@]+"${cargo_jobs[@]}"}" \ + --release \ + --target "${linux_musl_target}" \ + -p openshell-sandbox \ + --bin openshell-sandbox + linux_sandbox_bin="${target_dir}/${linux_musl_target}/release/openshell-sandbox" +fi host_gateway_bin= guest_gateway_bin= if [ "${mode}" = host ]; then - echo "==> Building native host openshell-gateway" - mise x -- cargo build "${cargo_jobs[@]}" \ - -p openshell-server \ - --bin openshell-gateway \ - --features bundled-z3 - host_gateway_bin="${target_dir}/debug/openshell-gateway" -else - echo "==> Preparing ${linux_gateway_rust_target} build target" - mise x -- rustup target add "${linux_gateway_rust_target}" >/dev/null - echo "==> Building Linux openshell-gateway (${linux_gateway_zig_target})" - ( - eval "$( - "${ROOT}/tasks/scripts/setup-zig-cc-wrapper.sh" \ - "${linux_gateway_zig_target}" \ - "${linux_gateway_zig_target}" \ - "${target_dir}/zig-gnu-wrapper/e2e" - )" - mise x -- cargo zigbuild "${cargo_jobs[@]}" \ - --release \ - --target "${linux_gateway_zig_target}" \ + if [ -n "${gateway_bin}" ]; then + echo "==> Using host openshell-gateway: ${gateway_bin}" + host_gateway_bin="${gateway_bin}" + else + echo "==> Building native host openshell-gateway" + mise x -- cargo build "${cargo_jobs[@]+"${cargo_jobs[@]}"}" \ -p openshell-server \ --bin openshell-gateway \ --features bundled-z3 - ) - guest_gateway_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-gateway" + host_gateway_bin="${target_dir}/debug/openshell-gateway" + fi +else + if [ -n "${gateway_bin}" ]; then + echo "==> Using Linux openshell-gateway: ${gateway_bin}" + guest_gateway_bin="${gateway_bin}" + else + echo "==> Preparing ${linux_gateway_rust_target} build target" + mise x -- rustup target add "${linux_gateway_rust_target}" >/dev/null + echo "==> Building Linux openshell-gateway (${linux_gateway_zig_target})" + ( + eval "$( + "${ROOT}/tasks/scripts/setup-zig-cc-wrapper.sh" \ + "${linux_gateway_zig_target}" \ + "${linux_gateway_zig_target}" \ + "${target_dir}/zig-gnu-wrapper/e2e" + )" + mise x -- cargo zigbuild "${cargo_jobs[@]+"${cargo_jobs[@]}"}" \ + --release \ + --target "${linux_gateway_zig_target}" \ + -p openshell-server \ + --bin openshell-gateway \ + --features bundled-z3 + ) + guest_gateway_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-gateway" + fi fi -expected_binaries=("${host_cli_bin}" "${linux_sandbox_bin}") -if [ "${mode}" = host ]; then - expected_binaries+=("${host_gateway_bin}") +expected_binaries=("${linux_sandbox_bin}") +if [ "${tests_in_vm}" -eq 1 ]; then + expected_binaries+=("${cli_bin}" "${guest_gateway_bin}") +elif [ "${mode}" = host ]; then + expected_binaries+=("${cli_bin}" "${host_gateway_bin}") else - expected_binaries+=("${guest_gateway_bin}") + expected_binaries+=("${cli_bin}" "${guest_gateway_bin}") fi for binary in "${expected_binaries[@]}"; do if [ ! -x "${binary}" ]; then @@ -300,6 +378,8 @@ supervisor_archive="${run_dir}/supervisor.tar" mkdir -p "${supervisor_rootfs}" install -m 0555 "${linux_sandbox_bin}" "${supervisor_rootfs}/openshell-sandbox" tar -C "${supervisor_rootfs}" -cf "${supervisor_archive}" openshell-sandbox +chmod 0644 "${supervisor_archive}" +test_artifacts=() child_pid= runtime_log= keep=0 @@ -321,6 +401,65 @@ start_child() { child_pid=$! } +build_e2e_test_artifacts() { + local build_log="${run_dir}/e2e-test-build.jsonl" + local artifacts_file="${run_dir}/e2e-test-artifacts.txt" + local build_args=( + mise x -- cargo zigbuild + --manifest-path e2e/rust/Cargo.toml + --features "${e2e_features}" + --target "${linux_gateway_zig_target}" + --message-format=json + ) + if [ -n "${suite_name}" ]; then + build_args+=(--test "${suite_name}") + else + build_args+=(--tests) + fi + + echo "==> Prebuilding E2E test artifacts for guest execution (${linux_gateway_rust_target})" + if ! ( + eval "$( + "${ROOT}/tasks/scripts/setup-zig-cc-wrapper.sh" \ + "${linux_gateway_zig_target}" \ + "${linux_gateway_zig_target}" \ + "${target_dir}/zig-gnu-wrapper/e2e-tests" + )" + "${build_args[@]}" + ) >"${build_log}"; then + echo "=== E2E test artifact build output ===" >&2 + cat "${build_log}" >&2 + echo "=== end E2E test artifact build output ===" >&2 + return 1 + fi + python3 - "${build_log}" >"${artifacts_file}" <<'PY' +import json +import sys + +for line in open(sys.argv[1], encoding="utf-8"): + try: + message = json.loads(line) + except json.JSONDecodeError: + continue + if message.get("reason") != "compiler-artifact": + continue + target = message.get("target") or {} + if "test" not in (target.get("kind") or []): + continue + executable = message.get("executable") + if executable: + print(executable) +PY + while IFS= read -r artifact; do + if [ -n "${artifact}" ]; then + test_artifacts+=("${artifact}") + fi + done <"${artifacts_file}" + if [ "${#test_artifacts[@]}" -eq 0 ]; then + die "cargo did not report any E2E test executables" + fi +} + # Invoked by the EXIT trap through cleanup. # shellcheck disable=SC2329 stop_child() { @@ -367,6 +506,10 @@ trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM +if [ "${tests_in_vm}" -eq 1 ]; then + build_e2e_test_artifacts +fi + jwt_source_dir="${run_dir}/gateway-jwt" host_runtime_dir= if [ "${mode}" = host ]; then @@ -390,7 +533,7 @@ gateway_name="openshell-e2e-${mode}-${host_port}" gateway_endpoint="http://127.0.0.1:${host_port}" export OPENSHELL_GATEWAY_ENDPOINT="${gateway_endpoint}" export OPENSHELL_GATEWAY="${gateway_name}" -export OPENSHELL_BIN="${host_cli_bin}" +export OPENSHELL_BIN="${cli_bin}" if [ "${mode}" = host ]; then case "${gateway_driver}" in @@ -424,6 +567,23 @@ else guest_launcher="${run_dir}/launch-gateway.sh" guest_launcher_path=/home/openshell/.cache/openshell-e2e/bin/launch-gateway guest_supervisor_archive_path=/home/openshell/.cache/openshell-e2e/supervisor.tar + guest_test_artifact_dir=/home/openshell/.cache/openshell-e2e/tests + guest_test_manifest="${run_dir}/test-artifacts.txt" + guest_test_manifest_path=/home/openshell/.cache/openshell-e2e/test-artifacts.txt + if [ "${tests_in_vm}" -eq 1 ]; then + : >"${guest_test_manifest}" + for artifact in "${test_artifacts[@]}"; do + printf '%s/%s\n' "${guest_test_artifact_dir}" "${artifact##*/}" >>"${guest_test_manifest}" + done + chmod 0644 "${guest_test_manifest}" + fi + guest_e2e_network_name="$(mise x -- python3 - "${gateway_config}" <<'PY' +import sys, tomllib + +config = tomllib.load(open(sys.argv[1], "rb")) +print(config.get("openshell", {}).get("drivers", {}).get("podman", {}).get("network_name", "openshell-e2e")) +PY +)" config_payload="$(base64 <"${gateway_config}" | tr -d '\r\n')" jwt_signing_payload="$(base64 <"${jwt_source_dir}/signing.pem" | tr -d '\r\n')" jwt_public_payload="$(base64 <"${jwt_source_dir}/public.pem" | tr -d '\r\n')" @@ -475,13 +635,178 @@ podman) esac report_timing "${gateway_driver} supervisor import" "\${phase_started_at}" cd /home/openshell + +if [ '${tests_in_vm}' = 1 ]; then + gateway_log=\${state_root}/gateway.log + gateway_pid_file=\${state_root}/gateway.pid + gateway_args_file=\${state_root}/gateway.args + spiffe_root=\${state_root}/spiffe + mkdir -p "\${spiffe_root}" "${guest_test_artifact_dir}" + + toml_string() { + python3 - "\$1" <<'PY' +import json +import sys + +print(json.dumps(sys.argv[1])) +PY + } + + pick_free_port() { + python3 - <<'PY' +import socket + +sock = socket.socket() +sock.bind(("0.0.0.0", 0)) +print(sock.getsockname()[1]) +sock.close() +PY + } + + insert_podman_config_key() { + local key=\$1 + local value=\$2 + + python3 - "\${config_path}" "\${key}" "\${value}" <<'PY' +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +key = sys.argv[2] +value = sys.argv[3] +section = "[openshell.drivers.podman]" +lines = path.read_text(encoding="utf-8").splitlines() +try: + start = next(index for index, line in enumerate(lines) if line.strip() == section) +except StopIteration: + raise SystemExit(f"{section} not found in {path}") +end = len(lines) +for index in range(start + 1, len(lines)): + if lines[index].lstrip().startswith("["): + end = index + break +for line in lines[start + 1:end]: + if line.split("=", 1)[0].strip() == key: + raise SystemExit(0) +lines.insert(end, f"{key} = {value}") +path.write_text("\\n".join(lines) + "\\n", encoding="utf-8") +PY + } + + write_gateway_args_file() { + : >"\${gateway_args_file}" + for arg in "\$@"; do + printf '%s\0' "\${arg}" >>"\${gateway_args_file}" + done + } + + stop_gateway() { + local gateway_pid= + if [ -f "\${gateway_pid_file}" ]; then + gateway_pid=\$(cat "\${gateway_pid_file}" 2>/dev/null || true) + fi + if [ -n "\${gateway_pid}" ] && kill -0 "\${gateway_pid}" 2>/dev/null; then + kill "\${gateway_pid}" 2>/dev/null || true + for _ in \$(seq 1 60); do + kill -0 "\${gateway_pid}" 2>/dev/null || break + sleep 0.5 + done + kill -KILL "\${gateway_pid}" 2>/dev/null || true + wait "\${gateway_pid}" 2>/dev/null || true + fi + rm -f "\${gateway_pid_file}" 2>/dev/null || true + } + + cleanup_guest_tests() { + local status=\$? + trap - EXIT INT TERM + stop_gateway + if [ "\${status}" -ne 0 ] && [ -f "\${gateway_log}" ]; then + echo "=== guest gateway log ===" >&2 + cat "\${gateway_log}" >&2 + echo "=== end guest gateway log ===" >&2 + fi + exit "\${status}" + } + trap cleanup_guest_tests EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + + export OPENSHELL_BIN=/usr/local/bin/openshell + export OPENSHELL_GATEWAY_ENDPOINT=http://127.0.0.1:${guest_port} + export OPENSHELL_GATEWAY=openshell-e2e-vm-${guest_port} + export OPENSHELL_PROVISION_TIMEOUT=\${OPENSHELL_PROVISION_TIMEOUT:-300} + export OPENSHELL_E2E_TESTS_IN_VM=1 + if [ '${gateway_driver}' = podman ]; then + export CONTAINER_ENGINE=podman + export OPENSHELL_E2E_DRIVER=podman + export OPENSHELL_E2E_NETWORK_NAME='${guest_e2e_network_name}' + export OPENSHELL_E2E_SANDBOX_NAMESPACE='${guest_e2e_network_name}' + export XDG_RUNTIME_DIR="\${XDG_RUNTIME_DIR:-/run/user/\$(id -u)}" + export OPENSHELL_PODMAN_SOCKET="\${XDG_RUNTIME_DIR}/podman/podman.sock" + export CONTAINER_HOST="unix://\${OPENSHELL_PODMAN_SOCKET}" + export OPENSHELL_E2E_CONTAINER_ENGINE_UNSET_XDG_CONFIG_HOME=1 + insert_podman_config_key socket_path "\$(toml_string "\${OPENSHELL_PODMAN_SOCKET}")" + insert_podman_config_key enable_bind_mounts true + + provider_spiffe_port=\$(pick_free_port) + export OPENSHELL_E2E_GATEWAY_SPIFFE_SOCKET="\${spiffe_root}/gateway.sock" + export OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET="\${OPENSHELL_E2E_GATEWAY_SPIFFE_SOCKET}" + export OPENSHELL_E2E_PROVIDER_SPIFFE_LISTEN="0.0.0.0:\${provider_spiffe_port}" + export OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET="tcp:169.254.1.2:\${provider_spiffe_port}" + insert_podman_config_key provider_spiffe_workload_api_socket "\$(toml_string "\${OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET}")" + fi + + gateway_args=( + --config "\${config_path}" + --bind-address 127.0.0.1 + --port ${guest_port} + --disable-tls + ) + write_gateway_args_file "\${gateway_args[@]}" + export OPENSHELL_E2E_GATEWAY_BIN=/usr/local/bin/openshell-gateway + export OPENSHELL_E2E_GATEWAY_ARGS_FILE="\${gateway_args_file}" + export OPENSHELL_E2E_GATEWAY_LOG="\${gateway_log}" + export OPENSHELL_E2E_GATEWAY_PID_FILE="\${gateway_pid_file}" + + /usr/local/bin/openshell-gateway "\${gateway_args[@]}" >"\${gateway_log}" 2>&1 & + printf '%s\n' "\$!" >"\${gateway_pid_file}" + + echo "==> Waiting for guest gateway readiness" + gateway_ready=0 + for _ in \$(seq 1 "${gateway_ready_timeout}"); do + if ! kill -0 "\$(cat "\${gateway_pid_file}")" 2>/dev/null; then + echo "ERROR: guest gateway exited before becoming ready" >&2 + exit 1 + fi + if NO_COLOR=1 /usr/local/bin/openshell status >/tmp/openshell-e2e-status.log 2>&1 && + grep -q "Connected" /tmp/openshell-e2e-status.log; then + gateway_ready=1 + break + fi + sleep 1 + done + if [ "\${gateway_ready}" -ne 1 ]; then + echo "ERROR: guest gateway did not become ready" >&2 + cat /tmp/openshell-e2e-status.log >&2 || true + exit 1 + fi + + while IFS= read -r test_bin <&3; do + [ -n "\${test_bin}" ] || continue + echo "==> Running guest E2E artifact: \${test_bin##*/}" + "\${test_bin}" --nocapture Running prebuilt E2E test artifacts inside ${vm} test guest" + "${vm_args[@]}" + exit $? + fi + echo "==> Starting ${vm} test guest gateway at ${gateway_endpoint}" start_child "${ROOT}" "${runtime_log}" "${vm_args[@]}" fi @@ -588,10 +926,17 @@ wait_for_gateway() { wait_for_gateway -echo "==> Running E2E suite: ${suite_name}" +test_args=( + cargo test + --manifest-path e2e/rust/Cargo.toml + --features "${e2e_features}" +) +echo "==> Running E2E features: ${e2e_features}" +if [ -n "${suite_name}" ]; then + echo "==> Running E2E suite: ${suite_name}" + test_args+=(--test "${suite_name}") +fi +test_args+=(-- --nocapture) + cd "${ROOT}" -cargo test \ - --manifest-path e2e/rust/Cargo.toml \ - --features e2e \ - --test "${suite_name}" \ - -- --nocapture +"${test_args[@]}" diff --git a/e2e/rust/tests/live_policy_update.rs b/e2e/rust/tests/live_policy_update.rs index 7a1e12923a..4cded2354b 100644 --- a/e2e/rust/tests/live_policy_update.rs +++ b/e2e/rust/tests/live_policy_update.rs @@ -56,6 +56,11 @@ ENV OPENSHELL_POLICY_POLL_INTERVAL_SECS=1 CMD ["sleep", "infinity"] "#; +const SPARSE_POLICY: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../examples/policy-advisor/sandbox-policy.yaml" +)); + // --------------------------------------------------------------------------- // Policy YAML builders // --------------------------------------------------------------------------- @@ -146,6 +151,15 @@ landlock: Ok(file) } +fn write_sparse_policy() -> Result { + let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; + file.write_all(SPARSE_POLICY.as_bytes()) + .map_err(|e| format!("write temp policy file: {e}"))?; + file.flush() + .map_err(|e| format!("flush temp policy file: {e}"))?; + Ok(file) +} + #[cfg(feature = "e2e-docker")] fn write_local_override_image() -> Result { let dir = tempfile::tempdir().map_err(|e| format!("create image context: {e}"))?; @@ -521,18 +535,18 @@ async fn live_policy_update_from_empty_network_policies() { /// no revision remaining `Pending` once the acknowledgement lands. #[tokio::test] async fn initial_sparse_policy_is_acknowledged_as_loaded() { - // Repo-relative path to the sparse network-only policy fixture. - let sparse_policy = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../examples/policy-advisor/sandbox-policy.yaml" - ); + let sparse_policy = write_sparse_policy().expect("write sparse policy fixture"); + let sparse_policy_path = sparse_policy + .path() + .to_str() + .expect("sparse policy path is not UTF-8"); let mut guard = SandboxGuard::create_keep_with_args( &[ "--name", "e2e-sparse-enrich", "--policy", - sparse_policy, + sparse_policy_path, "--no-tty", ], &["sh", "-c", "echo Ready && sleep infinity"], diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 1027654641..aef5a39d87 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -141,6 +141,13 @@ async fn rootless_podman_musl_getaddrinfo_uses_udp_policy_dns() { if !is_e2e_driver("podman") { return; } + if std::env::var_os("OPENSHELL_E2E_TESTS_IN_VM").is_some() { + eprintln!( + "skipping musl DNS probe test in guest prebuilt-artifact mode; \ + restore with a prebuilt probe artifact tracked in #3009" + ); + return; + } let probe = MuslDnsProbe::build().expect("build static musl DNS probe"); let fixture = SupportContainer::start_python( diff --git a/tasks/test.toml b/tasks/test.toml index 29e06b826a..49e8df7f1a 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -136,6 +136,10 @@ run = [ "CONTAINER_RUNTIME=docker e2e/with-keycloak.sh env OPENSHELL_E2E_OIDC_GATEWAY=1 e2e/with-docker-gateway.sh uv run pytest -m 'not gpu' e2e/python/oidc", ] +["e2e:podman:rootless"] +description = "Run Rust Podman e2e inside a rootless Ubuntu 26.04 Nix test guest" +run = "e2e/run.sh --vm ubuntu-26-04 --with podman-rootless --tests-in-vm --gateway-config e2e/configs/gateway/podman.toml --features e2e-podman" + ["e2e:podman:gpu"] description = "Run GPU e2e against a standalone gateway with the Podman compute driver" env = { OPENSHELL_E2E_PODMAN_GPU = "1", OPENSHELL_E2E_PODMAN_TEST = "gpu", OPENSHELL_E2E_PODMAN_FEATURES = "e2e-podman-gpu" } From bfc7e865ee5858d6333c48c9a5de153622305f7f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 31 Aug 2026 12:08:10 +0200 Subject: [PATCH 07/11] ci(e2e): run rootless Podman guest lane Signed-off-by: Evan Lezar --- .github/actions/setup-e2e-podman/action.yml | 112 ------------------- .github/actions/setup-e2e-sandbox/action.yml | 27 +++++ .github/workflows/branch-e2e.yml | 38 +------ .github/workflows/e2e-podman-test.yml | 86 +++++++------- .github/workflows/release-dev.yml | 5 +- .github/workflows/release-tag.yml | 3 +- 6 files changed, 73 insertions(+), 198 deletions(-) delete mode 100644 .github/actions/setup-e2e-podman/action.yml create mode 100644 .github/actions/setup-e2e-sandbox/action.yml diff --git a/.github/actions/setup-e2e-podman/action.yml b/.github/actions/setup-e2e-podman/action.yml deleted file mode 100644 index 33a54518da..0000000000 --- a/.github/actions/setup-e2e-podman/action.yml +++ /dev/null @@ -1,112 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: Setup E2E Podman -description: Configure the supported rootless Podman environment for E2E tests - -inputs: - mise-version: - description: mise release to install - required: false - default: v2026.4.25 - podman-major: - description: Expected Podman major version - required: true - podman-package-version: - description: Exact Ubuntu Podman package version - required: true - conmon-package-version: - description: Exact Ubuntu conmon package version - required: true - -runs: - using: composite - steps: - - uses: ./.github/actions/setup-mise - with: - version: ${{ inputs.mise-version }} - - - name: Install Podman and build dependencies - shell: bash - run: | # zizmor: ignore[github-env] persists only literal and trusted runner-derived paths - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - apparmor \ - build-essential \ - fuse-overlayfs \ - libssl-dev \ - openssh-client \ - passt \ - pkg-config \ - "conmon=${INPUTS_CONMON_PACKAGE_VERSION}" \ - "podman=${INPUTS_PODMAN_PACKAGE_VERSION}" \ - uidmap - # Hosted runners can place newer binaries under /usr/local. Select the - # distro CLI and use Podman's supported final conmon-path override. - podman_config="${RUNNER_TEMP}/openshell-containers.conf" - printf '%s\n' \ - '[engine]' \ - 'conmon_path = ["/usr/bin/conmon"]' \ - > "${podman_config}" - echo "/usr/bin" >> "${GITHUB_PATH}" - echo "CONTAINERS_CONF_OVERRIDE=${podman_config}" >> "${GITHUB_ENV}" - env: - INPUTS_CONMON_PACKAGE_VERSION: ${{ inputs.conmon-package-version }} - INPUTS_PODMAN_PACKAGE_VERSION: ${{ inputs.podman-package-version }} - - - name: Allow pasta to receive Podman stop signals - shell: bash - run: | - set -euo pipefail - # Ubuntu's packaged profile blocks this signal and makes Podman wait - # for its SIGKILL fallback. - profile=/etc/apparmor.d/usr.bin.pasta - rule=' signal (receive) peer=podman,' - if ! sudo grep -Fqx "${rule}" "${profile}"; then - sudo sed -i '\|^ include $|a\ signal (receive) peer=podman,' "${profile}" - fi - sudo grep -Fqx "${rule}" "${profile}" - sudo apparmor_parser --replace "${profile}" - - - name: Configure rootless Podman - shell: bash - run: | # zizmor: ignore[github-env] runtime path is derived solely from the runner UID - set -euo pipefail - if ! grep -q "^${USER}:" /etc/subuid; then - sudo usermod --add-subuids 100000-165535 "$USER" - fi - if ! grep -q "^${USER}:" /etc/subgid; then - sudo usermod --add-subgids 100000-165535 "$USER" - fi - runtime_dir="/run/user/$(id -u)" - sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" "$runtime_dir" - echo "XDG_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" - - - name: Verify rootless Podman environment - shell: bash - run: | - set -euo pipefail - podman_version="$(podman version --format '{{.Client.Version}}')" - case "$podman_version" in - "${INPUTS_PODMAN_MAJOR}".*) ;; - *) echo "ERROR: expected Podman ${INPUTS_PODMAN_MAJOR}.x, found $podman_version" >&2; exit 1 ;; - esac - test "$(dpkg-query -W -f='${Version}' podman)" = "${INPUTS_PODMAN_PACKAGE_VERSION}" - test "$(dpkg-query -W -f='${Version}' conmon)" = "${INPUTS_CONMON_PACKAGE_VERSION}" - test "$(command -v podman)" = "/usr/bin/podman" - test "$(podman info --format '{{.Host.Conmon.Path}}')" = "/usr/bin/conmon" - test "$(podman info --format '{{.Host.Security.Rootless}}')" = "true" - test "$(podman info --format '{{.Host.RootlessNetworkCmd}}')" = "pasta" - test "$(sudo sysctl -n kernel.apparmor_restrict_unprivileged_userns)" = "1" - echo "=== host ===" - uname -a - echo "=== AppArmor ===" - cat /proc/self/attr/current - sudo aa-status || true - echo "=== Podman ===" - podman version - podman info --debug - env: - INPUTS_PODMAN_MAJOR: ${{ inputs.podman-major }} - INPUTS_PODMAN_PACKAGE_VERSION: ${{ inputs.podman-package-version }} - INPUTS_CONMON_PACKAGE_VERSION: ${{ inputs.conmon-package-version }} diff --git a/.github/actions/setup-e2e-sandbox/action.yml b/.github/actions/setup-e2e-sandbox/action.yml new file mode 100644 index 0000000000..943bd75618 --- /dev/null +++ b/.github/actions/setup-e2e-sandbox/action.yml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Setup E2E Sandbox +description: Download an architecture-matched prebuilt OpenShell sandbox binary for E2E tests + +runs: + using: composite + steps: + - name: Download prebuilt sandbox + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ runner.arch == 'X64' && 'openshell-sandbox-x86_64-unknown-linux-musl' || 'openshell-sandbox-aarch64-unknown-linux-musl' }} + path: .e2e/prebuilt-sandbox + + - name: Configure prebuilt sandbox + shell: bash + run: | # zizmor: ignore[github-env] persists only a trusted runner-derived path + set -euo pipefail + sandbox="$GITHUB_WORKSPACE/.e2e/prebuilt-sandbox/openshell-sandbox" + if [[ ! -f "$sandbox" ]]; then + echo "downloaded artifact is missing $sandbox" >&2 + exit 1 + fi + chmod +x "$sandbox" + "$sandbox" --version + echo "OPENSHELL_SANDBOX_BIN=$sandbox" >> "$GITHUB_ENV" diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 0b050d0e51..419dc011cb 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -168,22 +168,6 @@ jobs: interpreter: /lib/ld-linux-aarch64.so.1 secrets: inherit - build-driver-podman: - needs: [pr_metadata, version] - if: needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - contents: read - uses: ./.github/workflows/build-binaries.yml - with: - package: openshell-driver-podman - binary: openshell-driver-podman - triple: x86_64-unknown-linux-gnu - runner: linux-amd64-cpu8 - dev-shell: .#devShells.x86_64-linux.glibc-2-28 - cargo-version: ${{ needs.version.outputs.cargo }} - interpreter: /lib64/ld-linux-x86-64.so.2 - secrets: inherit - build-driver-kubernetes: needs: [pr_metadata, version] if: needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -253,16 +237,13 @@ jobs: runner: linux-arm64-cpu8 podman-e2e: - needs: [pr_metadata, build-cli, build-gateway, build-supervisor-image] + needs: [pr_metadata, build-cli, build-gateway, build-sandbox] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read contents: read packages: read uses: ./.github/workflows/e2e-podman-test.yml - with: - image-tag: ${{ github.sha }} - vm-e2e: needs: [pr_metadata, build-cli, build-gateway, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -288,21 +269,6 @@ jobs: suite-matrix: >- [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}] - podman-external-driver-e2e: - needs: [pr_metadata, build-cli, build-gateway-plain, build-driver-podman, build-supervisor-image] - if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' - permissions: - actions: read - contents: read - packages: read - uses: ./.github/workflows/e2e-podman-test.yml - with: - image-tag: ${{ github.sha }} - gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu - external-driver-binary: openshell-driver-podman - suite-matrix: >- - [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}] - vm-external-driver-e2e: needs: [pr_metadata, build-cli, build-gateway-plain, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -428,7 +394,7 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, docker-e2e, podman-e2e, vm-e2e, docker-external-driver-e2e, podman-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] + needs: [pr_metadata, docker-e2e, podman-e2e, vm-e2e, docker-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: diff --git a/.github/workflows/e2e-podman-test.yml b/.github/workflows/e2e-podman-test.yml index 4a5a6d38d7..74f83b6c5f 100644 --- a/.github/workflows/e2e-podman-test.yml +++ b/.github/workflows/e2e-podman-test.yml @@ -6,9 +6,6 @@ name: Podman E2E Test on: workflow_call: inputs: - image-tag: - required: true - type: string checkout-ref: required: false type: string @@ -17,15 +14,6 @@ on: required: false type: string default: "" - external-driver-binary: - required: false - type: string - default: "" - suite-matrix: - required: false - type: string - default: >- - [{"suite":"provider-refresh-keycloak","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:provider-refresh-keycloak"}] permissions: actions: read @@ -34,18 +22,14 @@ permissions: jobs: e2e: - name: E2E (rust-podman-${{ matrix.suite }}, ${{ matrix.runner }}) - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: ${{ fromJSON(inputs.suite-matrix) }} + name: E2E (rust-podman-rootless, Ubuntu 26.04 Nix VM) + runs-on: ubuntu-26.04 + # Run rootless Podman inside a Nix-managed Ubuntu guest so Podman, pasta, + # and user-namespace setup are provisioned by versioned repository tooling + # rather than mutable hosted-runner packages. + timeout-minutes: 60 env: - IMAGE_TAG: ${{ inputs.image-tag }} MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell - OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -58,34 +42,48 @@ jobs: with: artifact-name: ${{ inputs.gateway-artifact }} - - if: inputs.external-driver-binary != '' - uses: ./.github/actions/setup-e2e-driver + - uses: ./.github/actions/setup-e2e-sandbox + + - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1 with: - binary: ${{ inputs.external-driver-binary }} + github_access_token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/setup-e2e-podman + - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17 with: - podman-major: ${{ matrix.podman_major }} - podman-package-version: ${{ matrix.podman_package_version }} - conmon-package-version: ${{ matrix.conmon_package_version }} + name: openshell - - name: Log in to GHCR - run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin + - name: Install mise + run: | + curl https://mise.run | MISE_VERSION=v2026.4.25 sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH" - - name: Run tests - run: ${{ matrix.cmd }} + - name: Install tools + run: mise install --locked - - name: Print AppArmor denials - if: always() - run: sudo dmesg | grep -E 'apparmor=.*DENIED|profile="unprivileged_userns"' | tail -100 || true + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + cmake \ + libclang-dev \ + libssl-dev \ + libz3-dev \ + openssh-client \ + pkg-config - - name: Fail on pasta SIGTERM AppArmor denial - if: always() + - name: Run tests run: | set -euo pipefail - denials="$(sudo dmesg | grep -E 'profile="pasta".*requested_mask="receive".*signal=term.*peer="podman"' || true)" - if [ -n "${denials}" ]; then - echo "::error::pasta denied Podman's SIGTERM; Podman will use its SIGKILL fallback" - printf '%s\n' "${denials}" - exit 1 - fi + + mise x -- e2e/run.sh \ + --vm ubuntu-26-04 \ + --with podman-rootless \ + --tests-in-vm \ + --cli-bin "$OPENSHELL_BIN" \ + --gateway-bin "$OPENSHELL_GATEWAY_BIN" \ + --sandbox-bin "$OPENSHELL_SANDBOX_BIN" \ + --gateway-config e2e/configs/gateway/podman.toml \ + --features e2e-podman diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 7982a81643..c756f2c5e2 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -136,15 +136,12 @@ jobs: runner: linux-arm64-cpu8 podman-e2e: - needs: [build-cli, build-gateway, build-supervisor-image] + needs: [build-cli, build-gateway, build-sandbox] permissions: actions: read contents: read packages: read uses: ./.github/workflows/e2e-podman-test.yml - with: - image-tag: ${{ github.sha }} - vm-e2e: needs: [build-cli, build-gateway, build-vm-driver] permissions: diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 92eb4adf14..a2f819d19f 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -165,14 +165,13 @@ jobs: runner: linux-arm64-cpu8 podman-e2e: - needs: [compute-versions, build-cli, build-gateway, build-supervisor-image] + needs: [compute-versions, build-cli, build-gateway, build-sandbox] permissions: actions: read contents: read packages: read uses: ./.github/workflows/e2e-podman-test.yml with: - image-tag: ${{ needs.compute-versions.outputs.source_sha }} checkout-ref: ${{ inputs.tag || github.ref }} vm-e2e: From fd4bbb74a0c7fc95360a39da3d65b4eedaa7b0ea Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 31 Aug 2026 16:20:15 +0200 Subject: [PATCH 08/11] test(podman): bound rootless E2E teardown Signed-off-by: Evan Lezar --- e2e/configs/gateway/podman.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/configs/gateway/podman.toml b/e2e/configs/gateway/podman.toml index c1549cd933..48e84a4bde 100644 --- a/e2e/configs/gateway/podman.toml +++ b/e2e/configs/gateway/podman.toml @@ -26,3 +26,5 @@ image_pull_policy = "missing" network_name = "openshell-e2e" grpc_endpoint = "http://host.containers.internal:8080" supervisor_image = "localhost/openshell/supervisor:e2e-vm" +# Keep rootless Podman E2E teardown bounded. +stop_timeout_secs = 15 From 015d751135e6a55ef7c8c8628bfec4345d8c4d7c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 27 Aug 2026 15:54:29 +0200 Subject: [PATCH 09/11] ci(e2e): add fedora rootful podman lane Signed-off-by: Evan Lezar --- .agents/skills/test-release-canary/SKILL.md | 37 +++++++- .github/workflows/branch-e2e.yml | 70 ++++++++++++++- .github/workflows/release-canary.yml | 86 +++++++------------ TESTING.md | 6 ++ e2e/configs/gateway/podman-rootful.toml | 29 +++++++ e2e/run.sh | 47 ++++++++-- nix/test-guest/README.md | 20 +++-- nix/test-guest/cache.sh | 2 +- .../configuration/podman-rootful.yml | 52 +++++++++++ nix/test-guest/default.nix | 1 + nix/test-guest/run.sh | 2 +- tasks/test.toml | 4 + 12 files changed, 278 insertions(+), 78 deletions(-) create mode 100644 e2e/configs/gateway/podman-rootful.toml create mode 100644 nix/test-guest/configuration/podman-rootful.yml diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md index 5e8bbf394c..ad000d25be 100644 --- a/.agents/skills/test-release-canary/SKILL.md +++ b/.agents/skills/test-release-canary/SKILL.md @@ -13,7 +13,7 @@ The Release Canary (`.github/workflows/release-canary.yml`) smoke-tests the arti |---|---|---| | `macos` | `macos-latest-xlarge` | `install.sh` resolves the Homebrew formula, brew installs the cask, and `openshell status` reaches the brew-services–backed local gateway with the VM driver. | | `ubuntu` | `ubuntu-latest` | `install.sh` installs the Debian package, the post-install systemd user service starts, and `openshell status` reaches the local gateway with the Docker driver. | -| `fedora` | `fedora:latest` container | `install.sh` installs the RPM packages, the local gateway starts under Podman, and `openshell status` succeeds. | +| `fedora` | `linux-amd64-cpu8` + Fedora Nix VM | `install.sh` installs the RPM packages, the root-owned local gateway starts with rootful Podman, and `openshell status` succeeds. | | `kubernetes` | `ubuntu-latest` + kind | `helm install oci://ghcr.io/nvidia/openshell/helm-chart --version 0.0.0-dev` succeeds in a kind cluster, the gateway pod becomes Ready, port-forward exposes 8080, and the released CLI registers the in-cluster gateway and runs `openshell status` against it. | All canary jobs disable anonymous OpenShell telemetry. Host package jobs inject @@ -116,6 +116,41 @@ Swap `0.0.0-dev` for `0.0.0-dev.` to pin to a specific dev build. Tear down Loopback registration auto-derives the gateway name to `openshell` if `--name` is omitted, which collides with the `install.sh`-installed local gateway — always pass `--name kind` (or another distinct name) when registering in addition to a local install. +## Local Fedora reproduction + +The `fedora` job uses the repository's Nix test-guest harness instead of +running Fedora inside Docker. It can be reproduced on a Linux host with Nix, +KVM, and the repository checkout: + +```shell +export INSTALL_SH_URL="https://raw.githubusercontent.com/NVIDIA/OpenShell/$(git rev-parse HEAD)/install.sh" +nix run .#test-guest -- \ + --distro fedora \ + --with podman-rootful \ + -- \ + sudo env \ + SUDO_USER=root \ + HOME=/root \ + XDG_RUNTIME_DIR=/run/user/0 \ + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \ + OPENSHELL_TELEMETRY_ENABLED=false \ + INSTALL_SH_URL="$INSTALL_SH_URL" \ + bash -s <<'EOF' +set -euo pipefail +mkdir -p "${XDG_RUNTIME_DIR}" +chmod 700 "${XDG_RUNTIME_DIR}" +systemctl start user-runtime-dir@0.service || true +systemctl start user@0.service +systemctl --user daemon-reload +mkdir -p "${HOME}/.config/openshell" +printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_PODMAN_SOCKET=/run/podman/podman.sock\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" +podman --url unix:///run/podman/podman.sock info +curl -LsSf "${INSTALL_SH_URL}" | sh +openshell status +EOF +``` + ## Diagnosing failures | Symptom | Likely cause | Where to look | diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 419dc011cb..77e90bfead 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -244,6 +244,74 @@ jobs: contents: read packages: read uses: ./.github/workflows/e2e-podman-test.yml + + podman-fedora-rootful-e2e: + name: E2E (rust-podman-rootful, Fedora Nix VM) + needs: [pr_metadata, build-cli, build-gateway, build-sandbox] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + runs-on: ubuntu-26.04 + timeout-minutes: 60 + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/setup-e2e-cli + + - uses: ./.github/actions/setup-e2e-gateway + + - uses: ./.github/actions/setup-e2e-sandbox + + - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + + - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17 + with: + name: openshell + + - name: Install mise + run: | + curl https://mise.run | MISE_VERSION=v2026.4.25 sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH" + + - name: Install tools + run: mise install --locked + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + cmake \ + libclang-dev \ + libssl-dev \ + libz3-dev \ + openssh-client \ + pkg-config + + - name: Run Fedora rootful Podman E2E + run: | + set -euo pipefail + + mise x -- e2e/run.sh \ + --vm fedora \ + --with podman-rootful \ + --guest-gateway-user root \ + --cli-bin "$OPENSHELL_BIN" \ + --gateway-bin "$OPENSHELL_GATEWAY_BIN" \ + --sandbox-bin "$OPENSHELL_SANDBOX_BIN" \ + --gateway-config e2e/configs/gateway/podman-rootful.toml \ + --features e2e-podman \ + --suite sandbox_lifecycle vm-e2e: needs: [pr_metadata, build-cli, build-gateway, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -394,7 +462,7 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, docker-e2e, podman-e2e, vm-e2e, docker-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] + needs: [pr_metadata, docker-e2e, podman-e2e, podman-fedora-rootful-e2e, vm-e2e, docker-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index 937e774db7..1507e41e4a 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -61,53 +61,43 @@ jobs: name: Fedora RPM if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} runs-on: linux-amd64-cpu8 - timeout-minutes: 20 + timeout-minutes: 30 env: - FEDORA_CANARY_CONTAINER: openshell-fedora-canary-${{ github.run_id }}-${{ github.run_attempt }} + INSTALL_SH_URL: https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh steps: - - name: Start Fedora systemd container and root user manager - run: | - set -euo pipefail + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - docker run --detach \ - --name "${FEDORA_CANARY_CONTAINER}" \ - --privileged \ - --cgroupns=host \ - --tmpfs /run \ - --tmpfs /tmp \ - --volume /sys/fs/cgroup:/sys/fs/cgroup:rw \ - fedora:latest \ - bash -lc 'dnf install -y curl dbus-daemon podman systemd && exec /usr/sbin/init' - - for _ in $(seq 1 120); do - if docker exec "${FEDORA_CANARY_CONTAINER}" systemctl list-units --no-pager >/dev/null 2>&1; then - break - fi - if [ "$(docker inspect -f '{{.State.Running}}' "${FEDORA_CANARY_CONTAINER}")" != "true" ]; then - echo "::error::Fedora systemd container exited before systemd became reachable" - docker logs "${FEDORA_CANARY_CONTAINER}" >&2 || true - exit 1 - fi - sleep 1 - done + - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} - if ! docker exec "${FEDORA_CANARY_CONTAINER}" systemctl list-units --no-pager >/dev/null 2>&1; then - echo "::error::Fedora systemd container did not become reachable within 120s" - docker logs "${FEDORA_CANARY_CONTAINER}" >&2 || true - exit 1 - fi + - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17 + with: + name: openshell + + - name: Install RPM in Fedora rootful Podman VM and check status + run: | + set -euo pipefail - docker exec --interactive "${FEDORA_CANARY_CONTAINER}" env \ + nix run .#test-guest -- \ + --distro fedora \ + --with podman-rootful \ + -- \ + sudo env \ + SUDO_USER=root \ HOME=/root \ XDG_RUNTIME_DIR=/run/user/0 \ DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \ + OPENSHELL_TELEMETRY_ENABLED="$OPENSHELL_TELEMETRY_ENABLED" \ + INSTALL_SH_URL="$INSTALL_SH_URL" \ bash -s <<'EOF' set -euo pipefail - # install.sh manages the RPM gateway as a systemd user unit. This - # container is booted with systemd as PID 1, but it still has no - # login session. Start root's user manager explicitly so the - # installer can test service restart and gateway registration - # instead of its "restart later" fallback. + + # install.sh manages the RPM gateway as a systemd user unit. Start + # root's user manager explicitly so the canary exercises the rootful + # service path on a real Fedora VM. mkdir -p "${XDG_RUNTIME_DIR}" chmod 700 "${XDG_RUNTIME_DIR}" systemctl start user-runtime-dir@0.service || true @@ -125,33 +115,15 @@ jobs: systemctl --user status --no-pager >&2 || true exit 1 fi - EOF - - - name: Install and check status - run: | - set -euo pipefail - docker exec --interactive "${FEDORA_CANARY_CONTAINER}" env \ - HOME=/root \ - XDG_RUNTIME_DIR=/run/user/0 \ - DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \ - OPENSHELL_TELEMETRY_ENABLED="$OPENSHELL_TELEMETRY_ENABLED" \ - INSTALL_SH_URL="https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh" \ - bash -s <<'EOF' - set -euo pipefail mkdir -p "${HOME}/.config/openshell" - printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_PODMAN_SOCKET=/run/podman/podman.sock\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" - podman info + podman --url unix:///run/podman/podman.sock info curl -LsSf "${INSTALL_SH_URL}" | sh openshell status EOF - - name: Stop Fedora systemd container - if: always() - run: | - docker rm -f "${FEDORA_CANARY_CONTAINER}" >/dev/null 2>&1 || true - ubuntu-snap: name: Ubuntu Snap if: ${{ github.event.workflow_run.conclusion == 'success' }} diff --git a/TESTING.md b/TESTING.md index 94e457d6ba..3643ca7d22 100644 --- a/TESTING.md +++ b/TESTING.md @@ -181,6 +181,12 @@ Run the rootless Podman suite in an Ubuntu 26.04 Nix test guest: mise run e2e:podman:rootless ``` +Run the focused rootful Podman suite in a Fedora Nix test guest: + +```shell +mise run e2e:podman:fedora-rootful +``` + Run the VM-backed Rust CLI e2e suite: ```shell diff --git a/e2e/configs/gateway/podman-rootful.toml b/e2e/configs/gateway/podman-rootful.toml new file mode 100644 index 0000000000..57bf093ed0 --- /dev/null +++ b/e2e/configs/gateway/podman-rootful.toml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[openshell] +version = 1 + +[openshell.gateway] +bind_address = "127.0.0.1:8080" +log_level = "info" +compute_drivers = ["podman"] +disable_tls = true + +[openshell.gateway.auth] +allow_unauthenticated_users = true + +[openshell.gateway.gateway_jwt] +signing_key_path = ".cache/openshell-e2e/gateway-jwt/signing.pem" +public_key_path = ".cache/openshell-e2e/gateway-jwt/public.pem" +kid_path = ".cache/openshell-e2e/gateway-jwt/kid" +gateway_id = "openshell-e2e" +ttl_secs = 0 + +[openshell.drivers.podman] +socket_path = "/run/podman/podman.sock" +default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +image_pull_policy = "missing" +network_name = "openshell-e2e" +grpc_endpoint = "http://host.containers.internal:8080" +supervisor_image = "localhost/openshell/supervisor:e2e-vm" diff --git a/e2e/run.sh b/e2e/run.sh index bf364f6176..bba07e3e8b 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -30,6 +30,8 @@ Options: --cli-bin PATH Use a prebuilt openshell CLI instead of building it --gateway-bin PATH Use a prebuilt openshell-gateway instead of building it --sandbox-bin PATH Use a prebuilt openshell-sandbox instead of building it + --guest-gateway-user USER + Run the VM guest gateway as USER (openshell or root) --gateway-config PATH Fully resolved gateway TOML --features FEATURES Rust e2e feature set to enable (default: e2e) @@ -103,6 +105,7 @@ gateway_config= gateway_bin= cli_bin= sandbox_bin= +guest_gateway_user=openshell e2e_features=e2e suite_name= tests_in_vm=0 @@ -139,6 +142,11 @@ while [ "$#" -gt 0 ]; do sandbox_bin="$(resolve_file "$2")" || die "--sandbox-bin does not name a file: $2" shift 2 ;; + --guest-gateway-user) + require_value "$1" "$#" "${2:-}" + guest_gateway_user=$2 + shift 2 + ;; --gateway-config) require_value "$1" "$#" "${2:-}" gateway_config=$2 @@ -184,6 +192,10 @@ print(tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"]["compute_dri if [ -z "${e2e_features}" ]; then die "--features must not be empty" fi +case "${guest_gateway_user}" in +openshell | root) ;; +*) die "--guest-gateway-user must be 'openshell' or 'root'" ;; +esac if [ -n "${suite_name}" ]; then if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9_-]*$ ]]; then die "suite name must contain only lowercase letters, digits, underscores, and hyphens: ${suite_name}" @@ -534,6 +546,7 @@ gateway_endpoint="http://127.0.0.1:${host_port}" export OPENSHELL_GATEWAY_ENDPOINT="${gateway_endpoint}" export OPENSHELL_GATEWAY="${gateway_name}" export OPENSHELL_BIN="${cli_bin}" +export OPENSHELL_E2E_DRIVER="${gateway_driver}" if [ "${mode}" = host ]; then case "${gateway_driver}" in @@ -627,10 +640,17 @@ docker) "${supervisor_image}" >/dev/null ;; podman) - podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" import \ - --change 'ENTRYPOINT ["/openshell-sandbox"]' \ - "${guest_supervisor_archive_path}" \ - "${supervisor_image}" >/dev/null + if [ '${guest_gateway_user}' = root ]; then + sudo podman --url unix:///run/podman/podman.sock import \ + --change 'ENTRYPOINT ["/openshell-sandbox"]' \ + "${guest_supervisor_archive_path}" \ + "${supervisor_image}" >/dev/null + else + podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" import \ + --change 'ENTRYPOINT ["/openshell-sandbox"]' \ + "${guest_supervisor_archive_path}" \ + "${supervisor_image}" >/dev/null + fi ;; esac report_timing "${gateway_driver} supervisor import" "\${phase_started_at}" @@ -800,11 +820,22 @@ PY exit 0 fi -exec /usr/local/bin/openshell-gateway \ - --config "\${config_path}" \ - --bind-address 127.0.0.1 \ - --port ${guest_port} \ +gateway_args=( + /usr/local/bin/openshell-gateway + --config "\${config_path}" + --bind-address 127.0.0.1 + --port ${guest_port} --disable-tls +) +if [ '${guest_gateway_user}' = root ]; then + exec sudo env \ + XDG_CONFIG_HOME="\${XDG_CONFIG_HOME}" \ + XDG_CACHE_HOME="\${XDG_CACHE_HOME}" \ + XDG_DATA_HOME="\${XDG_DATA_HOME}" \ + XDG_STATE_HOME="\${XDG_STATE_HOME}" \ + "\${gateway_args[@]}" +fi +exec "\${gateway_args[@]}" EOF chmod 0755 "${guest_launcher}" diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index a65baf02a3..418b50b349 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -40,6 +40,7 @@ nix/test-guest/ │ └── rocky.nix └── configuration/ ├── docker.yml + ├── podman-rootful.yml ├── podman-rootless.yml ├── podman.yml └── selinux.yml @@ -58,13 +59,13 @@ The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-gues ## Supported configurations -| Distro | Docker | Podman | Rootless Podman | SELinux | Package format | -| --- | --- | --- | --- | --- | --- | -| Ubuntu 24.04 | Yes | Yes | No | No | `.deb` | -| Ubuntu 26.04 | Yes | Yes | Yes | No | `.deb` | -| CentOS Stream 10 | No | Yes | No | Yes | `.rpm` | -| Fedora 44 | No | Yes | No | Yes | `.rpm` | -| Rocky Linux 9 | Yes | Yes | No | Yes | `.rpm` | +| Distro | Docker | Podman | Rootful Podman | Rootless Podman | SELinux | Package format | +| --- | --- | --- | --- | --- | --- | --- | +| Ubuntu 24.04 | Yes | Yes | No | No | No | `.deb` | +| Ubuntu 26.04 | Yes | Yes | No | Yes | No | `.deb` | +| CentOS Stream 10 | No | Yes | Yes | No | Yes | `.rpm` | +| Fedora 44 | No | Yes | Yes | No | Yes | `.rpm` | +| Rocky Linux 9 | Yes | Yes | Yes | No | Yes | `.rpm` | The `snapd` configuration is available for Ubuntu and prepares snapd for local Snap lifecycle experiments. It does not install Docker, because the Snap @@ -102,6 +103,7 @@ Other combinations use the same interface: nix run .#test-guest -- --distro rocky --with docker nix run .#test-guest -- --distro centos --with podman nix run .#test-guest -- --distro fedora --with podman +nix run .#test-guest -- --distro fedora --with podman-rootful nix run .#test-guest -- --distro ubuntu-26-04 --with podman-rootless ``` @@ -189,7 +191,7 @@ Cache command options: ```text --distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky ---with NAME Apply docker, podman, or selinux; repeatable +--with NAME Apply docker, podman, podman-rootful, podman-rootless, or selinux; repeatable --repository REF OCI repository without a tag --digest DIGEST Trusted OCI manifest digest required for pulls --cache-dir PATH Override the local prepared-disk cache directory @@ -264,7 +266,7 @@ The destination must be an absolute guest path. Copied files are installed with ```text --distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky ---with NAME Apply docker, podman, or selinux; repeatable +--with NAME Apply docker, podman, podman-rootful, podman-rootless, or selinux; repeatable --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy a regular file into the guest, preserving its host mode; repeatable diff --git a/nix/test-guest/cache.sh b/nix/test-guest/cache.sh index 35e3ac441c..f229143e47 100644 --- a/nix/test-guest/cache.sh +++ b/nix/test-guest/cache.sh @@ -13,7 +13,7 @@ Usage: Options: --distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky - --with NAME Apply a configuration; repeatable (docker, podman, podman-rootless, selinux) + --with NAME Apply a configuration; repeatable (docker, podman, podman-rootful, podman-rootless, selinux) --repository REF OCI repository without a tag --digest DIGEST Trusted OCI manifest digest required for pulls --cache-dir PATH Override the local prepared-disk cache directory diff --git a/nix/test-guest/configuration/podman-rootful.yml b/nix/test-guest/configuration/podman-rootful.yml new file mode 100644 index 0000000000..9947635f1a --- /dev/null +++ b/nix/test-guest/configuration/podman-rootful.yml @@ -0,0 +1,52 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# PROTOTYPE: Configure rootful Podman in a disposable test guest. + +- name: Configure rootful Podman + hosts: test_vm + become: true + gather_facts: true + + tasks: + - name: Validate rootful Podman support + ansible.builtin.assert: + that: + - ansible_facts.distribution in ["CentOS", "Fedora", "Rocky"] + fail_msg: >- + Rootful Podman guest coverage is supported on RPM-family guests, not + {{ ansible_facts.distribution }}. + + - name: Install rootful Podman + ansible.builtin.package: + name: podman + state: present + + - name: Enable the rootful Podman API socket + ansible.builtin.systemd_service: + name: podman.socket + enabled: true + state: started + + - name: Verify rootful Podman + ansible.builtin.command: + argv: + - podman + - --url + - unix:///run/podman/podman.sock + - info + changed_when: false + + - name: Verify rootful Podman mode + ansible.builtin.command: + argv: + - podman + - --url + - unix:///run/podman/podman.sock + - info + - --format + - "{% raw %}{{.Host.Security.Rootless}}{% endraw %}" + changed_when: false + register: podman_rootless + failed_when: podman_rootless.stdout != "false" diff --git a/nix/test-guest/default.nix b/nix/test-guest/default.nix index b2249e1f4e..a95901f733 100644 --- a/nix/test-guest/default.nix +++ b/nix/test-guest/default.nix @@ -28,6 +28,7 @@ let configurations = { docker = ./configuration/docker.yml; podman = ./configuration/podman.yml; + podman-rootful = ./configuration/podman-rootful.yml; podman-rootless = ./configuration/podman-rootless.yml; selinux = ./configuration/selinux.yml; snapd = ./configuration/snapd.yml; diff --git a/nix/test-guest/run.sh b/nix/test-guest/run.sh index 9e5d19baef..3bc954bf83 100644 --- a/nix/test-guest/run.sh +++ b/nix/test-guest/run.sh @@ -13,7 +13,7 @@ Usage: Options: --distro NAME Base distro: ubuntu-24-04, ubuntu-26-04, centos, fedora, or rocky - --with NAME Apply a configuration; repeatable (docker, podman, podman-rootless, selinux, snapd) + --with NAME Apply a configuration; repeatable (docker, podman, podman-rootful, podman-rootless, selinux, snapd) --install PATH Install a .deb or .rpm package; repeatable --copy SRC:DEST Copy a regular file to an absolute guest path, preserving its host mode; repeatable diff --git a/tasks/test.toml b/tasks/test.toml index 49e8df7f1a..cdd9db6001 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -140,6 +140,10 @@ run = [ description = "Run Rust Podman e2e inside a rootless Ubuntu 26.04 Nix test guest" run = "e2e/run.sh --vm ubuntu-26-04 --with podman-rootless --tests-in-vm --gateway-config e2e/configs/gateway/podman.toml --features e2e-podman" +["e2e:podman:fedora-rootful"] +description = "Run focused Podman e2e against a rootful Fedora Nix test guest" +run = "e2e/run.sh --vm fedora --with podman-rootful --guest-gateway-user root --gateway-config e2e/configs/gateway/podman-rootful.toml --features e2e-podman --suite sandbox_lifecycle" + ["e2e:podman:gpu"] description = "Run GPU e2e against a standalone gateway with the Podman compute driver" env = { OPENSHELL_E2E_PODMAN_GPU = "1", OPENSHELL_E2E_PODMAN_TEST = "gpu", OPENSHELL_E2E_PODMAN_FEATURES = "e2e-podman-gpu" } From d69d690d4b1c17aaae8806f1b198a6637ccd616c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 31 Aug 2026 13:08:45 +0200 Subject: [PATCH 10/11] test(e2e): generalize rootful podman guest setup Signed-off-by: Evan Lezar --- nix/test-guest/README.md | 5 +++-- nix/test-guest/configuration/podman-rootful.yml | 10 +--------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index 418b50b349..b82ec9eeba 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -61,8 +61,8 @@ The root [`flake.nix`](../../flake.nix) exposes this directory as the `test-gues | Distro | Docker | Podman | Rootful Podman | Rootless Podman | SELinux | Package format | | --- | --- | --- | --- | --- | --- | --- | -| Ubuntu 24.04 | Yes | Yes | No | No | No | `.deb` | -| Ubuntu 26.04 | Yes | Yes | No | Yes | No | `.deb` | +| Ubuntu 24.04 | Yes | Yes | Yes | No | No | `.deb` | +| Ubuntu 26.04 | Yes | Yes | Yes | Yes | No | `.deb` | | CentOS Stream 10 | No | Yes | Yes | No | Yes | `.rpm` | | Fedora 44 | No | Yes | Yes | No | Yes | `.rpm` | | Rocky Linux 9 | Yes | Yes | Yes | No | Yes | `.rpm` | @@ -104,6 +104,7 @@ nix run .#test-guest -- --distro rocky --with docker nix run .#test-guest -- --distro centos --with podman nix run .#test-guest -- --distro fedora --with podman nix run .#test-guest -- --distro fedora --with podman-rootful +nix run .#test-guest -- --distro ubuntu-24-04 --with podman-rootful nix run .#test-guest -- --distro ubuntu-26-04 --with podman-rootless ``` diff --git a/nix/test-guest/configuration/podman-rootful.yml b/nix/test-guest/configuration/podman-rootful.yml index 9947635f1a..3ae5467fa4 100644 --- a/nix/test-guest/configuration/podman-rootful.yml +++ b/nix/test-guest/configuration/podman-rootful.yml @@ -7,17 +7,9 @@ - name: Configure rootful Podman hosts: test_vm become: true - gather_facts: true + gather_facts: false tasks: - - name: Validate rootful Podman support - ansible.builtin.assert: - that: - - ansible_facts.distribution in ["CentOS", "Fedora", "Rocky"] - fail_msg: >- - Rootful Podman guest coverage is supported on RPM-family guests, not - {{ ansible_facts.distribution }}. - - name: Install rootful Podman ansible.builtin.package: name: podman From 45dda2844c60532cdf26bb24b83e961bc5b26e08 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 31 Aug 2026 14:07:56 +0200 Subject: [PATCH 11/11] ci(e2e): smoke RPM install with rootful podman Signed-off-by: Evan Lezar --- .github/workflows/branch-e2e.yml | 105 +++++++++++++++----------- .github/workflows/e2e-podman-test.yml | 69 ++++++++++++++--- 2 files changed, 119 insertions(+), 55 deletions(-) diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 77e90bfead..232b7a4a56 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -123,6 +123,17 @@ jobs: cargo-version: ${{ needs.version.outputs.cargo }} secrets: inherit + build-rpm: + needs: [pr_metadata, version, build-cli, build-gateway] + if: needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + contents: read + uses: ./.github/workflows/rpm-package.yml + with: + checkout-ref: ${{ github.sha }} + cargo-version: ${{ needs.version.outputs.cargo }} + secrets: inherit + build-gateway-plain: needs: [pr_metadata, version] if: needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -246,28 +257,37 @@ jobs: uses: ./.github/workflows/e2e-podman-test.yml podman-fedora-rootful-e2e: - name: E2E (rust-podman-rootful, Fedora Nix VM) needs: [pr_metadata, build-cli, build-gateway, build-sandbox] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read contents: read packages: read - runs-on: ubuntu-26.04 - timeout-minutes: 60 - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: ./.github/workflows/e2e-podman-test.yml + with: + test-name: rust-podman-rootful, Fedora Nix VM + distro: fedora + configuration: podman-rootful + guest-gateway-user: root + gateway-config: e2e/configs/gateway/podman-rootful.toml + tests-in-vm: false + suite: sandbox_lifecycle + + podman-fedora-rootful-rpm: + name: RPM install (rootful Podman, Fedora Nix VM) + needs: [pr_metadata, build-rpm] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + runs-on: linux-amd64-cpu8 + timeout-minutes: 45 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: ./.github/actions/setup-e2e-cli - - - uses: ./.github/actions/setup-e2e-gateway - - - uses: ./.github/actions/setup-e2e-sandbox - - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1 with: github_access_token: ${{ secrets.GITHUB_TOKEN }} @@ -276,42 +296,41 @@ jobs: with: name: openshell - - name: Install mise - run: | - curl https://mise.run | MISE_VERSION=v2026.4.25 sh - echo "$HOME/.local/bin" >> "$GITHUB_PATH" - echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH" - - - name: Install tools - run: mise install --locked + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: rpm-linux-x86_64 + path: package-input - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - build-essential \ - clang \ - cmake \ - libclang-dev \ - libssl-dev \ - libz3-dev \ - openssh-client \ - pkg-config - - - name: Run Fedora rootful Podman E2E + - name: Install PR RPMs and start rootful Podman gateway run: | set -euo pipefail - mise x -- e2e/run.sh \ - --vm fedora \ + nix run .#test-guest -- \ + --distro fedora \ --with podman-rootful \ - --guest-gateway-user root \ - --cli-bin "$OPENSHELL_BIN" \ - --gateway-bin "$OPENSHELL_GATEWAY_BIN" \ - --sandbox-bin "$OPENSHELL_SANDBOX_BIN" \ - --gateway-config e2e/configs/gateway/podman-rootful.toml \ - --features e2e-podman \ - --suite sandbox_lifecycle + --install package-input/openshell-[0-9]*.rpm \ + --install package-input/openshell-gateway-*.rpm \ + -- \ + sudo env \ + SUDO_USER=root \ + HOME=/root \ + XDG_RUNTIME_DIR=/run/user/0 \ + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \ + OPENSHELL_TELEMETRY_ENABLED=false \ + bash -s <<'EOF' + set -euo pipefail + mkdir -p "${XDG_RUNTIME_DIR}" + chmod 700 "${XDG_RUNTIME_DIR}" + systemctl start user-runtime-dir@0.service || true + systemctl start user@0.service + systemctl --user daemon-reload + mkdir -p "${HOME}/.config/openshell" + printf 'OPENSHELL_DRIVERS=podman\nOPENSHELL_PODMAN_SOCKET=/run/podman/podman.sock\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \ + "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env" + podman --url unix:///run/podman/podman.sock info + systemctl --user enable --now openshell-gateway + openshell status + EOF vm-e2e: needs: [pr_metadata, build-cli, build-gateway, build-vm-driver] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -462,7 +481,7 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, docker-e2e, podman-e2e, podman-fedora-rootful-e2e, vm-e2e, docker-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] + needs: [pr_metadata, docker-e2e, podman-e2e, podman-fedora-rootful-e2e, podman-fedora-rootful-rpm, vm-e2e, docker-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: diff --git a/.github/workflows/e2e-podman-test.yml b/.github/workflows/e2e-podman-test.yml index 74f83b6c5f..beb4244837 100644 --- a/.github/workflows/e2e-podman-test.yml +++ b/.github/workflows/e2e-podman-test.yml @@ -14,6 +14,34 @@ on: required: false type: string default: "" + test-name: + required: false + type: string + default: rust-podman-rootless, Ubuntu 26.04 Nix VM + distro: + required: false + type: string + default: ubuntu-26-04 + configuration: + required: false + type: string + default: podman-rootless + guest-gateway-user: + required: false + type: string + default: openshell + gateway-config: + required: false + type: string + default: e2e/configs/gateway/podman.toml + tests-in-vm: + required: false + type: boolean + default: true + suite: + required: false + type: string + default: "" permissions: actions: read @@ -22,11 +50,10 @@ permissions: jobs: e2e: - name: E2E (rust-podman-rootless, Ubuntu 26.04 Nix VM) + name: E2E (${{ inputs.test-name }}) runs-on: ubuntu-26.04 - # Run rootless Podman inside a Nix-managed Ubuntu guest so Podman, pasta, - # and user-namespace setup are provisioned by versioned repository tooling - # rather than mutable hosted-runner packages. + # Run Podman inside a Nix-managed guest so runtime setup is provisioned by + # versioned repository tooling rather than mutable hosted-runner packages. timeout-minutes: 60 env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -75,15 +102,33 @@ jobs: pkg-config - name: Run tests + env: + TEST_GUEST_DISTRO: ${{ inputs.distro }} + TEST_GUEST_CONFIGURATION: ${{ inputs.configuration }} + TESTS_IN_VM: ${{ inputs.tests-in-vm }} + GUEST_GATEWAY_USER: ${{ inputs.guest-gateway-user }} + GATEWAY_CONFIG: ${{ inputs.gateway-config }} + E2E_SUITE: ${{ inputs.suite }} run: | set -euo pipefail - mise x -- e2e/run.sh \ - --vm ubuntu-26-04 \ - --with podman-rootless \ - --tests-in-vm \ - --cli-bin "$OPENSHELL_BIN" \ - --gateway-bin "$OPENSHELL_GATEWAY_BIN" \ - --sandbox-bin "$OPENSHELL_SANDBOX_BIN" \ - --gateway-config e2e/configs/gateway/podman.toml \ + args=( + mise x -- e2e/run.sh + --vm "$TEST_GUEST_DISTRO" + --with "$TEST_GUEST_CONFIGURATION" + --cli-bin "$OPENSHELL_BIN" + --gateway-bin "$OPENSHELL_GATEWAY_BIN" + --sandbox-bin "$OPENSHELL_SANDBOX_BIN" + --gateway-config "$GATEWAY_CONFIG" --features e2e-podman + ) + if [ "$TESTS_IN_VM" = true ]; then + args+=(--tests-in-vm) + fi + if [ "$GUEST_GATEWAY_USER" != openshell ]; then + args+=(--guest-gateway-user "$GUEST_GATEWAY_USER") + fi + if [ -n "$E2E_SUITE" ]; then + args+=(--suite "$E2E_SUITE") + fi + "${args[@]}"