From 69241907d6c22dacc9c1b93e0d9fdc24e9b6b4db Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 3 Sep 2026 18:19:21 +0200 Subject: [PATCH 1/4] refactor(conformance): compose scenarios through a trait Signed-off-by: Evan Lezar --- architecture/build.md | 13 +- crates/openshell-conformance-cli/src/main.rs | 29 +++-- crates/openshell-conformance/src/lib.rs | 112 ++++++++++++------ .../src/scenarios/sandbox_continuity.rs | 43 +++++-- .../src/scenarios/smoke.rs | 34 ++++-- 5 files changed, 158 insertions(+), 73 deletions(-) diff --git a/architecture/build.md b/architecture/build.md index 972f847bb9..aa33e4e702 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -257,11 +257,14 @@ and retain that provenance with the local entry; mutable tags are used only for explicit publication. CLI conformance runs after target provisioning. Action-free scenarios operate -only through the configured OpenShell CLI. A versioned conformance plan may add -an ordered sequence of target-supplied host-side actions, such as a gateway -restart, while the scenario remains responsible for black-box sandbox -continuity checks. The plan exposes opaque executable paths and timeouts rather -than driver or package-manager configuration; target setup owns those details. +only through the configured OpenShell CLI. A scenario may be a named +collection of internal leaf scenarios. Collections are selected and reported +as one scenario while their runner owns cleanup for every child. A versioned +conformance plan may add an ordered sequence of target-supplied host-side +actions, such as a gateway restart, while the scenario remains responsible for +black-box sandbox continuity checks. The plan exposes opaque executable paths +and timeouts rather than driver or package-manager configuration; target setup +owns those details. ## Python Wheel Packaging diff --git a/crates/openshell-conformance-cli/src/main.rs b/crates/openshell-conformance-cli/src/main.rs index c7f35bc13f..b1958841f0 100644 --- a/crates/openshell-conformance-cli/src/main.rs +++ b/crates/openshell-conformance-cli/src/main.rs @@ -103,15 +103,15 @@ fn list(output: OutputFormat) -> Result<(), String> { match output { OutputFormat::Text => { for candidate in scenarios() { - println!("{:<16} {}", candidate.name, candidate.description); + println!("{:<16} {}", candidate.name(), candidate.description()); } } OutputFormat::Json => { let result = scenarios() .iter() .map(|candidate| ScenarioDescription { - name: candidate.name, - description: candidate.description, + name: candidate.name(), + description: candidate.description(), }) .collect::>(); println!( @@ -137,7 +137,7 @@ async fn run( let selected = select_scenarios(requested)?; let mut results = Vec::with_capacity(selected.len()); for candidate in selected { - let plan_run = default_plan_run(candidate.name); + let plan_run = default_plan_run(candidate.name()); results.push(run_scenario(candidate, &plan_run, binary.as_ref(), None).await); } @@ -179,20 +179,20 @@ fn default_plan_run(scenario: &str) -> PlanRun { } async fn run_scenario( - candidate: &'static Scenario, + candidate: &'static dyn Scenario, plan_run: &PlanRun, binary: Option<&PathBuf>, host_action_executor: Option>, ) -> ScenarioResult<'static> { let runner = binary.map_or_else( - || OpenShellRunner::new(candidate.name), - |path| OpenShellRunner::with_binary(path.clone(), candidate.name), + || OpenShellRunner::new(candidate.name()), + |path| OpenShellRunner::with_binary(path.clone(), candidate.name()), ); let mut runner = match runner { Ok(runner) => runner, Err(error) => { return ScenarioResult { - name: candidate.name, + name: candidate.name(), passed: false, diagnostic: Some(error.to_string()), }; @@ -208,7 +208,7 @@ async fn run_scenario( }; let outcome = runner.finish(scenario_result).await; ScenarioResult { - name: candidate.name, + name: candidate.name(), passed: outcome.is_ok(), diagnostic: outcome.err(), } @@ -276,7 +276,7 @@ fn read_plan(path: &PathBuf) -> Result { ConformancePlan::parse(&contents).map_err(|error| format!("invalid conformance plan: {error}")) } -fn select_scenarios(requested: &[String]) -> Result, String> { +fn select_scenarios(requested: &[String]) -> Result, String> { if requested.is_empty() { return Ok(default_scenarios().collect()); } @@ -359,19 +359,22 @@ mod tests { #[test] fn selects_named_scenario() { let selected = select_scenarios(&["smoke".to_string()]).expect("select smoke"); - assert_eq!(selected[0].name, "smoke"); + assert_eq!(selected[0].name(), "smoke"); } #[test] fn unknown_scenario_has_actionable_diagnostic() { - let error = select_scenarios(&["missing".to_string()]).expect_err("unknown scenario"); + let error = select_scenarios(&["missing".to_string()]) + .err() + .expect("unknown scenario"); assert!(error.contains("openshell-conformance list")); } #[test] fn action_scenario_requires_an_explicit_plan() { let error = select_scenarios(&["sandbox-continuity".to_string()]) - .expect_err("action scenario requires a plan"); + .err() + .expect("action scenario requires a plan"); assert!(error.contains("requires an explicit --plan")); } diff --git a/crates/openshell-conformance/src/lib.rs b/crates/openshell-conformance/src/lib.rs index 63cd1677e4..77e2a233bb 100644 --- a/crates/openshell-conformance/src/lib.rs +++ b/crates/openshell-conformance/src/lib.rs @@ -27,39 +27,81 @@ use self::executor::{CliExecutionError, CliExecutor, ProcessCli}; pub use plan::{ConformancePlan, HostAction, PlanDiagnostics, PlanRun, WorkloadExpectation}; pub use scenarios::{SANDBOX_CONTINUITY_SCENARIO, SMOKE_SCENARIO}; -/// An installed conformance scenario. -#[derive(Debug)] -pub struct Scenario { - pub name: &'static str, - pub description: &'static str, - requires_plan: bool, - run: for<'a> fn(&'a mut OpenShellRunner, &'a PlanRun) -> ScenarioFuture<'a>, - validate_plan_run: Option, +pub type ScenarioFuture<'a> = Pin> + Send + 'a>>; + +/// A reusable `OpenShell` conformance contract. +pub trait Scenario: Send + Sync { + /// Stable command-line name for this scenario. + fn name(&self) -> &'static str; + + /// Human-readable summary for scenario discovery. + fn description(&self) -> &'static str; + + /// Whether this scenario may run only through an explicit target plan. + fn requires_plan(&self) -> bool { + false + } + + /// Whether this scenario is selected when no scenario names are supplied. + fn runs_by_default(&self) -> bool { + !self.requires_plan() + } + + /// Validates target-supplied inputs before the scenario starts. + fn validate_plan_run(&self, plan_run: &PlanRun) -> Result<(), String> { + default_validate_plan_run(plan_run) + } + + /// Execute this scenario with a suite-owned runner and target-supplied plan input. + fn run<'a>(&self, runner: &'a mut OpenShellRunner, plan_run: &'a PlanRun) + -> ScenarioFuture<'a>; } -pub type ScenarioFuture<'a> = Pin> + Send + 'a>>; -type PlanRunValidator = fn(&PlanRun) -> Result<(), String>; +/// A scenario that executes a fixed sequence of child scenarios. +pub struct ScenarioCollection { + name: &'static str, + description: &'static str, + scenarios: &'static [&'static dyn Scenario], +} -impl Scenario { - pub async fn run( - &self, - runner: &mut OpenShellRunner, - plan_run: &PlanRun, - ) -> Result<(), String> { - self.validate_plan_run(plan_run)?; - (self.run)(runner, plan_run).await +impl ScenarioCollection { + #[must_use] + pub const fn new( + name: &'static str, + description: &'static str, + scenarios: &'static [&'static dyn Scenario], + ) -> Self { + Self { + name, + description, + scenarios, + } } +} - pub fn validate_plan_run(&self, plan_run: &PlanRun) -> Result<(), String> { - self.validate_plan_run.map_or_else( - || default_validate_plan_run(plan_run), - |validate| validate(plan_run), - ) +impl Scenario for ScenarioCollection { + fn name(&self) -> &'static str { + self.name } - /// Whether this scenario may run only through an explicit target plan. - pub fn requires_plan(&self) -> bool { - self.requires_plan + fn description(&self) -> &'static str { + self.description + } + + fn run<'a>( + &self, + runner: &'a mut OpenShellRunner, + plan_run: &'a PlanRun, + ) -> ScenarioFuture<'a> { + let validation = self.validate_plan_run(plan_run); + let scenarios = self.scenarios; + Box::pin(async move { + validation?; + for scenario in scenarios { + scenario.run(runner, plan_run).await?; + } + Ok(()) + }) } } @@ -73,23 +115,27 @@ fn default_validate_plan_run(plan_run: &PlanRun) -> Result<(), String> { Ok(()) } -const SCENARIOS: &[Scenario] = &[SMOKE_SCENARIO, SANDBOX_CONTINUITY_SCENARIO]; +const SCENARIOS: &[&dyn Scenario] = &[SMOKE_SCENARIO, SANDBOX_CONTINUITY_SCENARIO]; /// Returns every scenario compiled into this distribution. -pub fn scenarios() -> &'static [Scenario] { +pub fn scenarios() -> &'static [&'static dyn Scenario] { SCENARIOS } -/// Finds a scenario by its stable command-line name. -pub fn scenario(name: &str) -> Option<&'static Scenario> { - scenarios().iter().find(|candidate| candidate.name == name) +/// Finds a publicly selectable scenario by its stable command-line name. +pub fn scenario(name: &str) -> Option<&'static dyn Scenario> { + scenarios() + .iter() + .copied() + .find(|candidate| candidate.name() == name) } /// Returns scenarios that need no host-level disruption capability. -pub fn default_scenarios() -> impl Iterator { +pub fn default_scenarios() -> impl Iterator { scenarios() .iter() - .filter(|scenario| !scenario.requires_plan) + .copied() + .filter(|scenario| scenario.runs_by_default()) } const CLEANUP_TIMEOUT: Duration = Duration::from_secs(120); diff --git a/crates/openshell-conformance/src/scenarios/sandbox_continuity.rs b/crates/openshell-conformance/src/scenarios/sandbox_continuity.rs index 7fe1f0d06b..b54e590a24 100644 --- a/crates/openshell-conformance/src/scenarios/sandbox_continuity.rs +++ b/crates/openshell-conformance/src/scenarios/sandbox_continuity.rs @@ -22,20 +22,39 @@ struct SandboxState { phase: String, } +struct SandboxContinuityScenario; + /// Certify sandbox state and workspace continuity across host-side actions. -pub const SANDBOX_CONTINUITY_SCENARIO: Scenario = Scenario { - name: "sandbox-continuity", - description: "Verify sandbox state and workspace continuity across planned host actions.", - requires_plan: true, - run: run_sandbox_continuity, - validate_plan_run: Some(validate_plan_run), -}; +pub static SANDBOX_CONTINUITY_SCENARIO: &dyn Scenario = &SandboxContinuityScenario; + +impl Scenario for SandboxContinuityScenario { + fn name(&self) -> &'static str { + "sandbox-continuity" + } + + fn description(&self) -> &'static str { + "Verify sandbox state and workspace continuity across planned host actions." + } -fn run_sandbox_continuity<'a>( - runner: &'a mut OpenShellRunner, - plan_run: &'a PlanRun, -) -> ScenarioFuture<'a> { - Box::pin(async move { run_sandbox_continuity_inner(runner, plan_run).await }) + fn requires_plan(&self) -> bool { + true + } + + fn validate_plan_run(&self, plan_run: &PlanRun) -> Result<(), String> { + validate_plan_run(plan_run) + } + + fn run<'a>( + &self, + runner: &'a mut OpenShellRunner, + plan_run: &'a PlanRun, + ) -> ScenarioFuture<'a> { + let validation = self.validate_plan_run(plan_run); + Box::pin(async move { + validation?; + run_sandbox_continuity_inner(runner, plan_run).await + }) + } } fn validate_plan_run(plan_run: &PlanRun) -> Result<(), String> { diff --git a/crates/openshell-conformance/src/scenarios/smoke.rs b/crates/openshell-conformance/src/scenarios/smoke.rs index cb5ca40122..18bf8f30ef 100644 --- a/crates/openshell-conformance/src/scenarios/smoke.rs +++ b/crates/openshell-conformance/src/scenarios/smoke.rs @@ -22,17 +22,31 @@ struct SandboxListEntry { phase: String, } +struct SmokeScenario; + /// Certify status -> create -> list Ready -> exec -> delete -> list empty. -pub const SMOKE_SCENARIO: Scenario = Scenario { - name: "smoke", - description: "Create, inspect, execute in, and delete a base sandbox.", - requires_plan: false, - run: run_smoke, - validate_plan_run: None, -}; - -fn run_smoke<'a>(runner: &'a mut OpenShellRunner, _plan_run: &'a PlanRun) -> ScenarioFuture<'a> { - Box::pin(async move { run_smoke_inner(runner).await }) +pub static SMOKE_SCENARIO: &dyn Scenario = &SmokeScenario; + +impl Scenario for SmokeScenario { + fn name(&self) -> &'static str { + "smoke" + } + + fn description(&self) -> &'static str { + "Create, inspect, execute in, and delete a base sandbox." + } + + fn run<'a>( + &self, + runner: &'a mut OpenShellRunner, + plan_run: &'a PlanRun, + ) -> ScenarioFuture<'a> { + let validation = self.validate_plan_run(plan_run); + Box::pin(async move { + validation?; + run_smoke_inner(runner).await + }) + } } async fn run_smoke_inner(runner: &mut OpenShellRunner) -> Result<(), String> { From 395477f7b62a4ed21afa0a62ac26bbc982b3a4c0 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 3 Sep 2026 18:26:44 +0200 Subject: [PATCH 2/4] test(conformance): add sandbox lifecycle scenario Signed-off-by: Evan Lezar --- crates/openshell-conformance/src/lib.rs | 8 +- .../src/scenarios/mod.rs | 2 + .../src/scenarios/sandbox_lifecycle.rs | 331 ++++++++++++++++++ 3 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 crates/openshell-conformance/src/scenarios/sandbox_lifecycle.rs diff --git a/crates/openshell-conformance/src/lib.rs b/crates/openshell-conformance/src/lib.rs index 77e2a233bb..de458ff2af 100644 --- a/crates/openshell-conformance/src/lib.rs +++ b/crates/openshell-conformance/src/lib.rs @@ -25,7 +25,7 @@ use tokio::time::sleep; use self::executor::{CliExecutionError, CliExecutor, ProcessCli}; pub use plan::{ConformancePlan, HostAction, PlanDiagnostics, PlanRun, WorkloadExpectation}; -pub use scenarios::{SANDBOX_CONTINUITY_SCENARIO, SMOKE_SCENARIO}; +pub use scenarios::{SANDBOX_CONTINUITY_SCENARIO, SANDBOX_LIFECYCLE_SCENARIO, SMOKE_SCENARIO}; pub type ScenarioFuture<'a> = Pin> + Send + 'a>>; @@ -115,7 +115,11 @@ fn default_validate_plan_run(plan_run: &PlanRun) -> Result<(), String> { Ok(()) } -const SCENARIOS: &[&dyn Scenario] = &[SMOKE_SCENARIO, SANDBOX_CONTINUITY_SCENARIO]; +const SCENARIOS: &[&dyn Scenario] = &[ + SMOKE_SCENARIO, + SANDBOX_LIFECYCLE_SCENARIO, + SANDBOX_CONTINUITY_SCENARIO, +]; /// Returns every scenario compiled into this distribution. pub fn scenarios() -> &'static [&'static dyn Scenario] { diff --git a/crates/openshell-conformance/src/scenarios/mod.rs b/crates/openshell-conformance/src/scenarios/mod.rs index c5211b3690..d430f63def 100644 --- a/crates/openshell-conformance/src/scenarios/mod.rs +++ b/crates/openshell-conformance/src/scenarios/mod.rs @@ -4,7 +4,9 @@ //! Registered, portable conformance scenarios. mod sandbox_continuity; +mod sandbox_lifecycle; mod smoke; pub use sandbox_continuity::SANDBOX_CONTINUITY_SCENARIO; +pub use sandbox_lifecycle::SANDBOX_LIFECYCLE_SCENARIO; pub use smoke::SMOKE_SCENARIO; diff --git a/crates/openshell-conformance/src/scenarios/sandbox_lifecycle.rs b/crates/openshell-conformance/src/scenarios/sandbox_lifecycle.rs new file mode 100644 index 0000000000..8f49879b89 --- /dev/null +++ b/crates/openshell-conformance/src/scenarios/sandbox_lifecycle.rs @@ -0,0 +1,331 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Portable sandbox lifecycle conformance scenarios. + +use std::time::Duration; + +use serde::Deserialize; + +use crate::{OpenShellRunner, PlanRun, Poll, Scenario, ScenarioCollection, ScenarioFuture}; + +const CREATE_TIMEOUT: Duration = Duration::from_secs(600); +const COMMAND_TIMEOUT: Duration = Duration::from_secs(120); +const TRANSITION_TIMEOUT: Duration = Duration::from_secs(240); +const TRANSITION_INTERVAL: Duration = Duration::from_secs(2); + +#[derive(Debug, Deserialize)] +struct SandboxState { + name: String, + phase: String, +} + +struct StopStartPreservesWorkspaceScenario; +struct StoppedCanBeDeletedScenario; + +static STOP_START_PRESERVES_WORKSPACE_SCENARIO: StopStartPreservesWorkspaceScenario = + StopStartPreservesWorkspaceScenario; +static STOPPED_CAN_BE_DELETED_SCENARIO: StoppedCanBeDeletedScenario = StoppedCanBeDeletedScenario; + +static SANDBOX_LIFECYCLE_CHILDREN: &[&dyn Scenario] = &[ + &STOP_START_PRESERVES_WORKSPACE_SCENARIO, + &STOPPED_CAN_BE_DELETED_SCENARIO, +]; + +static SANDBOX_LIFECYCLE_COLLECTION: ScenarioCollection = ScenarioCollection::new( + "sandbox-lifecycle", + "Verify sandbox stop, start, and deletion lifecycle behavior.", + SANDBOX_LIFECYCLE_CHILDREN, +); + +/// Certify portable sandbox lifecycle behavior as one conformance scenario. +pub static SANDBOX_LIFECYCLE_SCENARIO: &dyn Scenario = &SANDBOX_LIFECYCLE_COLLECTION; + +impl Scenario for StopStartPreservesWorkspaceScenario { + fn name(&self) -> &'static str { + "stop-start-preserves-workspace" + } + + fn description(&self) -> &'static str { + "Stopping and starting a sandbox preserves its workspace and restarts its main process." + } + + fn run<'a>( + &self, + runner: &'a mut OpenShellRunner, + plan_run: &'a PlanRun, + ) -> ScenarioFuture<'a> { + let validation = self.validate_plan_run(plan_run); + Box::pin(async move { + validation?; + stop_start_preserves_workspace(runner).await + }) + } +} + +impl Scenario for StoppedCanBeDeletedScenario { + fn name(&self) -> &'static str { + "stopped-can-be-deleted" + } + + fn description(&self) -> &'static str { + "A stopped sandbox can be deleted without being started again." + } + + fn run<'a>( + &self, + runner: &'a mut OpenShellRunner, + plan_run: &'a PlanRun, + ) -> ScenarioFuture<'a> { + let validation = self.validate_plan_run(plan_run); + Box::pin(async move { + validation?; + stopped_can_be_deleted(runner).await + }) + } +} + +async fn stop_start_preserves_workspace(runner: &mut OpenShellRunner) -> Result<(), String> { + let sandbox_name = format!("ct-{}-ss", runner.id()); + let sentinel = format!("openshell-stop-start-{}", runner.id()); + let sentinel_path = "/sandbox/.openshell-stop-start-sentinel"; + let run_count_path = "/sandbox/.openshell-main-run-count"; + let main = format!( + "count=0; test ! -f '{run_count_path}' || count=$(cat '{run_count_path}'); \ + count=$((count + 1)); printf '%s\\n' \"$count\" > '{run_count_path}'; \ + exec sleep infinity" + ); + + create_running_sandbox(runner, &sandbox_name, &main, "stop-start").await?; + exec_expect_exact( + runner, + &sandbox_name, + "write-sentinel", + &[ + "sh", + "-lc", + &format!("printf '%s\\n' '{sentinel}' > '{sentinel_path}'"), + ], + "", + ) + .await?; + + run_lifecycle_command(runner, "stop", &sandbox_name, "stop").await?; + wait_for_phase(runner, &sandbox_name, "Stopped", "stop-start/stopped").await?; + + let stopped_exec = runner + .step("stop-start/exec-while-stopped") + .description(format!( + "sandbox '{sandbox_name}' rejects exec while stopped" + )) + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + &sandbox_name, + "--no-tty", + "--", + "cat", + sentinel_path, + ]) + .await + .map_err(|error| error.to_string())?; + if stopped_exec.success() { + return Err( + stopped_exec.failure_diagnostic("sandbox exec fails while the sandbox is stopped") + ); + } + + run_lifecycle_command(runner, "start", &sandbox_name, "start").await?; + wait_for_phase(runner, &sandbox_name, "Ready", "stop-start/restarted").await?; + + exec_expect_exact( + runner, + &sandbox_name, + "read-sentinel", + &["cat", sentinel_path], + &format!("{sentinel}\n"), + ) + .await?; + exec_expect_exact( + runner, + &sandbox_name, + "read-main-run-count", + &["cat", run_count_path], + "2\n", + ) + .await +} + +async fn stopped_can_be_deleted(runner: &mut OpenShellRunner) -> Result<(), String> { + let sandbox_name = format!("ct-{}-sd", runner.id()); + create_running_sandbox( + runner, + &sandbox_name, + "exec sleep infinity", + "stopped-delete", + ) + .await?; + + run_lifecycle_command(runner, "stop", &sandbox_name, "stopped-delete/stop").await?; + wait_for_phase(runner, &sandbox_name, "Stopped", "stopped-delete/stopped").await?; + run_lifecycle_command(runner, "delete", &sandbox_name, "stopped-delete/delete").await?; + wait_for_absence(runner, &sandbox_name, "stopped-delete/deleted").await?; + runner.forget_sandbox(&sandbox_name); + Ok(()) +} + +async fn create_running_sandbox( + runner: &mut OpenShellRunner, + sandbox_name: &str, + main: &str, + step: &str, +) -> Result<(), String> { + runner.track_sandbox(sandbox_name); + let create = runner + .step(format!("{step}/create")) + .description(format!("sandbox '{sandbox_name}' is created")) + .with_timeout(CREATE_TIMEOUT) + .run(&[ + "sandbox", + "create", + "--name", + sandbox_name, + "--from", + "base", + "--detach", + "--no-tty", + "--", + "sh", + "-lc", + main, + ]) + .await + .map_err(|error| error.to_string())?; + create.require_success()?; + wait_for_phase(runner, sandbox_name, "Ready", &format!("{step}/ready")).await +} + +async fn run_lifecycle_command( + runner: &OpenShellRunner, + operation: &str, + sandbox_name: &str, + step: &str, +) -> Result<(), String> { + let result = runner + .step(step) + .description(format!("sandbox '{sandbox_name}' {operation} succeeds")) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", operation, sandbox_name]) + .await + .map_err(|error| error.to_string())?; + result.require_success() +} + +async fn exec_expect_exact( + runner: &OpenShellRunner, + sandbox_name: &str, + step: &str, + command: &[&str], + expected_stdout: &str, +) -> Result<(), String> { + let mut args = vec!["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]; + args.extend_from_slice(command); + let result = runner + .step(format!("stop-start/{step}")) + .description(format!("sandbox '{sandbox_name}' exec {step} succeeds")) + .with_timeout(COMMAND_TIMEOUT) + .run(&args) + .await + .map_err(|error| error.to_string())?; + result.require_success()?; + if result.stdout() == expected_stdout { + Ok(()) + } else { + Err(result.failure_diagnostic(&format!("stdout is exactly {expected_stdout:?}"))) + } +} + +async fn wait_for_phase( + runner: &mut OpenShellRunner, + sandbox_name: &str, + expected_phase: &str, + step: &str, +) -> Result<(), String> { + let sandbox_name = sandbox_name.to_string(); + let expected_phase = expected_phase.to_string(); + let step = step.to_string(); + let poll_step = step.clone(); + runner + .poll_until( + &poll_step, + TRANSITION_TIMEOUT, + TRANSITION_INTERVAL, + async move |runner| { + let result = runner + .step(format!("{step}/get")) + .description(format!( + "sandbox '{sandbox_name}' reaches phase {expected_phase}" + )) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "get", &sandbox_name, "--output", "json"]) + .await; + match result { + Ok(result) if !result.success() => { + Poll::Pending(result.failure_diagnostic(&format!( + "sandbox '{sandbox_name}' can be retrieved" + ))) + } + Ok(result) => match result.json::() { + Ok(state) if state.name != sandbox_name => Poll::Failed(format!( + "sandbox get returned {:?}; expected '{sandbox_name}'", + state.name + )), + Ok(state) if state.phase == expected_phase => Poll::Ready(()), + Ok(state) => Poll::Pending(format!( + "sandbox '{sandbox_name}' phase is {:?}; expected {expected_phase:?}", + state.phase + )), + Err(error) => Poll::Failed(error.to_string()), + }, + Err(error) => Poll::Pending(error.to_string()), + } + }, + ) + .await + .map_err(|error| error.to_string()) +} + +async fn wait_for_absence( + runner: &mut OpenShellRunner, + sandbox_name: &str, + step: &str, +) -> Result<(), String> { + let sandbox_name = sandbox_name.to_string(); + let step = step.to_string(); + let poll_step = step.clone(); + runner + .poll_until( + &poll_step, + TRANSITION_TIMEOUT, + TRANSITION_INTERVAL, + async move |runner| { + let result = runner + .step(format!("{step}/get")) + .description(format!("sandbox '{sandbox_name}' is no longer retrievable")) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "get", &sandbox_name, "--output", "json"]) + .await; + match result { + Ok(result) if !result.success() => Poll::Ready(()), + Ok(_) => { + Poll::Pending(format!("sandbox '{sandbox_name}' is still retrievable")) + } + Err(error) => Poll::Pending(error.to_string()), + } + }, + ) + .await + .map_err(|error| error.to_string()) +} From b52722a6642b3662db7f111d5731c38738e8b4f2 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 3 Sep 2026 18:33:08 +0200 Subject: [PATCH 3/4] test(conformance): run lifecycle checks before continuity Signed-off-by: Evan Lezar --- nix/test-guest/conformance-plans/gateway-upgrade-restart.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nix/test-guest/conformance-plans/gateway-upgrade-restart.toml b/nix/test-guest/conformance-plans/gateway-upgrade-restart.toml index fde903b778..ffbca9d3d9 100644 --- a/nix/test-guest/conformance-plans/gateway-upgrade-restart.toml +++ b/nix/test-guest/conformance-plans/gateway-upgrade-restart.toml @@ -10,6 +10,9 @@ timeout_secs = 60 [[runs]] scenario = "smoke" +[[runs]] +scenario = "sandbox-lifecycle" + [[runs]] scenario = "sandbox-continuity" workload_expectation = "reconciled" From 1b9a6029816df1c33308484417592019787224e5 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 4 Sep 2026 10:15:02 +0200 Subject: [PATCH 4/4] test(conformance): test candidate RPMs directly Signed-off-by: Evan Lezar --- .github/workflows/conformance.yml | 14 +++++------ crates/openshell-conformance/src/plan.rs | 9 +++++--- nix/test-guest/README.md | 6 +++-- .../conformance-plans/gateway-restart.toml | 23 +++++++++++++++++++ 4 files changed, 40 insertions(+), 12 deletions(-) create mode 100644 nix/test-guest/conformance-plans/gateway-restart.toml diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 775eb9f83e..68ead60b23 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -154,7 +154,7 @@ jobs: name: openshell-conformance-x86_64-unknown-linux-musl path: conformance-input - - name: Run RPM gateway continuity conformance + - name: Run candidate RPM gateway conformance shell: bash run: | set -euo pipefail @@ -173,11 +173,11 @@ jobs: --distro fedora \ --with podman-rootless \ --with selinux \ - --copy "${candidate_cli_package[0]}:/var/lib/openshell-conformance/candidate/openshell.rpm" \ - --copy "${candidate_gateway_package[0]}:/var/lib/openshell-conformance/candidate/openshell-gateway.rpm" \ + --install "${candidate_cli_package[0]}" \ + --install "${candidate_gateway_package[0]}" \ --copy conformance-input/openshell-conformance:/tmp/openshell-conformance \ - --copy nix/test-guest/conformance-plans/gateway-upgrade-restart.toml:/tmp/conformance-plan.toml \ - --provision openshell-rpm-latest-release \ + --copy nix/test-guest/conformance-plans/gateway-restart.toml:/tmp/conformance-plan.toml \ + --provision openshell-rpm \ --provision gateway-rootless-podman \ - --provision openshell-rpm-gateway-upgrade \ - -- /tmp/openshell-conformance run --plan /tmp/conformance-plan.toml + -- /home/openshell/.local/bin/openshell-test-guest-as-gateway-user \ + /tmp/openshell-conformance run --plan /tmp/conformance-plan.toml diff --git a/crates/openshell-conformance/src/plan.rs b/crates/openshell-conformance/src/plan.rs index ad955ea5f6..fd17e532ce 100644 --- a/crates/openshell-conformance/src/plan.rs +++ b/crates/openshell-conformance/src/plan.rs @@ -125,7 +125,7 @@ mod tests { use super::*; #[test] - fn parses_a_smoke_and_continuity_plan() { + fn parses_a_smoke_lifecycle_and_continuity_plan() { let plan = ConformancePlan::parse( r#" version = 1 @@ -133,6 +133,9 @@ mod tests { [[runs]] scenario = "smoke" + [[runs]] + scenario = "sandbox-lifecycle" + [[runs]] scenario = "sandbox-continuity" workload_expectation = "reconciled" @@ -145,8 +148,8 @@ mod tests { ) .expect("valid plan"); - assert_eq!(plan.runs.len(), 2); - assert_eq!(plan.runs[1].actions[0].name, "gateway-upgrade"); + assert_eq!(plan.runs.len(), 3); + assert_eq!(plan.runs[2].actions[0].name, "gateway-upgrade"); } #[test] diff --git a/nix/test-guest/README.md b/nix/test-guest/README.md index 0006a331fb..b73c6eab45 100644 --- a/nix/test-guest/README.md +++ b/nix/test-guest/README.md @@ -217,8 +217,10 @@ EOF `openshell-rpm` expects OpenShell to have been installed with `--install`. It uses the RPM-owned `/usr/bin` binaries and `openshell-gateway` user service, without copied development artifacts or a supervisor archive. Compose it with -`gateway-rootless-podman` before an RPM action such as -`openshell-rpm-gateway-upgrade`. +`gateway-rootless-podman` to test the installed candidate directly. The +`gateway-restart.toml` plan covers the candidate through smoke, lifecycle, and +gateway-restart continuity checks. `gateway-upgrade-restart.toml` retains the +separate upgrade action contract for dedicated upgrade testing. `openshell-rpm-latest-release` downloads and installs the latest stable OpenShell GitHub release for the guest architecture, then publishes the same diff --git a/nix/test-guest/conformance-plans/gateway-restart.toml b/nix/test-guest/conformance-plans/gateway-restart.toml new file mode 100644 index 0000000000..91904f0b33 --- /dev/null +++ b/nix/test-guest/conformance-plans/gateway-restart.toml @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version = 1 + +[diagnostics] +command = "/home/openshell/.local/bin/openshell-test-guest-diagnostics" +timeout_secs = 60 + +[[runs]] +scenario = "smoke" + +[[runs]] +scenario = "sandbox-lifecycle" + +[[runs]] +scenario = "sandbox-continuity" +workload_expectation = "reconciled" + +[[runs.actions]] +name = "gateway-restart" +command = "/home/openshell/.local/bin/openshell-test-guest-gateway-restart" +timeout_secs = 120