test(conformance): add plan-driven continuity verification

.github/workflows/conformance.yml

@@ -154,42 +154,30 @@ jobs:
           name: openshell-conformance-x86_64-unknown-linux-musl
           path: conformance-input
 
-      - name: Install RPMs and check status
+      - name: Run RPM gateway continuity conformance
         shell: bash
         run: |
           set -euo pipefail
           chmod +x conformance-input/openshell-conformance
-          guest_script="$RUNNER_TEMP/conformance.sh"
-          cat > "$guest_script" <<'EOF'
-          set -euo pipefail
-
-          on_exit() {
-            rc=$?
-            trap - EXIT
-            if [ "$rc" -ne 0 ]; then
-              systemctl --user status openshell-gateway --no-pager || true
-              journalctl --user -u openshell-gateway --no-pager -n 200 || true
-              podman info || true
-              podman ps --all || true
-              getenforce || true
-            fi
-            exit "$rc"
-          }
-          trap on_exit EXIT
-
-          mkdir -p "$HOME/.config/openshell"
-          echo 'OPENSHELL_TELEMETRY_ENABLED=false' > "$HOME/.config/openshell/gateway.env"
-          systemctl --user enable --now openshell-gateway
-          openshell gateway add --local https://127.0.0.1:17670 --name openshell
-          /tmp/openshell-conformance run smoke
-          EOF
+          shopt -s nullglob
+          candidate_cli_package=(rpm-input/openshell-[0-9]*.rpm)
+          candidate_gateway_package=(rpm-input/openshell-gateway-[0-9]*.rpm)
+          if [[ ${#candidate_cli_package[@]} -ne 1 || ${#candidate_gateway_package[@]} -ne 1 ]]; then
+            echo "expected one candidate CLI and gateway RPM" >&2
+            printf 'RPM artifacts:\n' >&2
+            printf '  %s\n' rpm-input/* >&2
+            exit 1
+          fi
 
           OPENSHELL_TEST_GUEST_CACHE_DISABLE=1 nix run .#test-guest -- \
             --distro fedora \
             --with podman-rootless \
             --with selinux \
-            --install rpm-input/openshell-[0-9]*.rpm \
-            --install rpm-input/openshell-gateway-[0-9]*.rpm \
-            --copy "$guest_script:/tmp/conformance.sh" \
+            --copy "${candidate_cli_package[0]}:/var/lib/openshell-conformance/candidate/openshell.rpm" \
+            --copy "${candidate_gateway_package[0]}:/var/lib/openshell-conformance/candidate/openshell-gateway.rpm" \
             --copy conformance-input/openshell-conformance:/tmp/openshell-conformance \
-            -- bash /tmp/conformance.sh
+            --copy nix/test-guest/conformance-plans/gateway-upgrade-restart.toml:/tmp/conformance-plan.toml \
+            --provision openshell-rpm-latest-release \
+            --provision gateway-rootless-podman \
+            --provision openshell-rpm-gateway-upgrade \
+            -- /tmp/openshell-conformance run --plan /tmp/conformance-plan.toml

architecture/build.md

@@ -256,6 +256,13 @@ pulls and explicit publication. OCI pulls require a trusted manifest digest
 and retain that provenance with the local entry; mutable tags are used only
 for explicit publication.
 
+CLI conformance runs after target provisioning. Action-free scenarios operate
+only through the configured OpenShell CLI. A versioned conformance plan may add
+an ordered sequence of target-supplied host-side actions, such as a gateway
+restart, while the scenario remains responsible for black-box sandbox
+continuity checks. The plan exposes opaque executable paths and timeouts rather
+than driver or package-manager configuration; target setup owns those details.
+
 ## Python Wheel Packaging
 
 The generated protobuf/gRPC stubs under `python/openshell/_proto/` are gitignored

crates/openshell-conformance-cli/src/main.rs

@@ -3,11 +3,19 @@
 
 //! Standalone runner for `OpenShell` CLI conformance scenarios.
 
+use std::fmt::Write;
+use std::future::Future;
+use std::io::Read;
 use std::path::PathBuf;
+use std::pin::Pin;
 use std::process::ExitCode;
+use std::sync::Arc;
 
 use clap::{Parser, Subcommand, ValueEnum};
-use openshell_conformance::{OpenShellRunner, Scenario, scenario, scenarios};
+use openshell_conformance::{
+    ConformancePlan, HostAction, HostActionExecutor, OpenShellRunner, PlanRun, Scenario,
+    default_scenarios, scenario, scenarios,
+};
 use serde::Serialize;
 
 #[derive(Debug, Parser)]
@@ -28,13 +36,16 @@ enum Command {
         #[arg(long, value_enum, default_value_t = OutputFormat::Text)]
         output: OutputFormat,
     },
-    /// Run all registered scenarios, or named scenarios.
+    /// Run action-free scenarios, named scenarios, or an explicit plan.
     Run {
-        /// Scenario names. Omit to run every registered scenario.
+        /// Action-free scenario names. Omit to run every action-free scenario.
         scenarios: Vec<String>,
         /// Explicit path to the `OpenShell` CLI. Defaults to `openshell` on PATH.
         #[arg(long)]
         openshell_bin: Option<PathBuf>,
+        /// Versioned TOML conformance plan. Use '-' to read the plan from stdin.
+        #[arg(long, conflicts_with = "scenarios")]
+        plan: Option<PathBuf>,
         #[arg(long, value_enum, default_value_t = OutputFormat::Text)]
         output: OutputFormat,
     },
@@ -82,8 +93,9 @@ async fn execute(cli: Cli) -> Result<(), String> {
         Command::Run {
             scenarios: requested,
             openshell_bin,
+            plan,
             output,
-        } => run(&requested, openshell_bin, output).await,
+        } => run(&requested, openshell_bin, plan, output).await,
     }
 }
 
@@ -114,39 +126,111 @@ fn list(output: OutputFormat) -> Result<(), String> {
 async fn run(
     requested: &[String],
     binary: Option<PathBuf>,
+    plan_path: Option<PathBuf>,
     output: OutputFormat,
 ) -> Result<(), String> {
+    if let Some(plan_path) = plan_path {
+        let plan = read_plan(&plan_path)?;
+        return run_plan(&plan, binary, output).await;
+    }
+
     let selected = select_scenarios(requested)?;
     let mut results = Vec::with_capacity(selected.len());
     for candidate in selected {
-        let runner = binary.as_ref().map_or_else(
-            || OpenShellRunner::new(candidate.name),
-            |path| OpenShellRunner::with_binary(path.clone(), candidate.name),
-        );
-        let mut runner = match runner {
-            Ok(runner) => runner,
-            Err(error) => {
-                results.push(ScenarioResult {
-                    name: candidate.name,
-                    passed: false,
-                    diagnostic: Some(error.to_string()),
-                });
-                continue;
-            }
-        };
-        eprintln!("CLI conformance run ID: {}", runner.id());
-        let scenario_result = match runner.check_gateway_status().await {
-            Ok(()) => candidate.run(&mut runner).await,
-            Err(error) => Err(error),
+        let plan_run = default_plan_run(candidate.name);
+        results.push(run_scenario(candidate, &plan_run, binary.as_ref(), None).await);
+    }
+
+    render_results(results, output)
+}
+
+async fn run_plan(
+    plan: &ConformancePlan,
+    binary: Option<PathBuf>,
+    output: OutputFormat,
+) -> Result<(), String> {
+    let executor: Arc<dyn HostActionExecutor> = Arc::new(ProcessHostAction);
+    let mut results = Vec::with_capacity(plan.runs.len());
+    for plan_run in &plan.runs {
+        let candidate = scenario(&plan_run.scenario)
+            .expect("validated conformance plan references a registered scenario");
+        let result =
+            run_scenario(candidate, plan_run, binary.as_ref(), Some(executor.clone())).await;
+        let result = match (result.passed, &plan.diagnostics) {
+            (false, Some(diagnostics)) => append_diagnostics(result, &executor, diagnostics).await,
+            _ => result,
         };
-        let outcome = runner.finish(scenario_result).await;
-        results.push(ScenarioResult {
-            name: candidate.name,
-            passed: outcome.is_ok(),
-            diagnostic: outcome.err(),
-        });
+        let passed = result.passed;
+        results.push(result);
+        if !passed {
+            break;
+        }
+    }
+
+    render_results(results, output)
+}
+
+fn default_plan_run(scenario: &str) -> PlanRun {
+    PlanRun {
+        scenario: scenario.to_string(),
+        workload_expectation: None,
+        actions: Vec::new(),
+    }
+}
+
+async fn run_scenario(
+    candidate: &'static Scenario,
+    plan_run: &PlanRun,
+    binary: Option<&PathBuf>,
+    host_action_executor: Option<Arc<dyn HostActionExecutor>>,
+) -> ScenarioResult<'static> {
+    let runner = binary.map_or_else(
+        || OpenShellRunner::new(candidate.name),
+        |path| OpenShellRunner::with_binary(path.clone(), candidate.name),
+    );
+    let mut runner = match runner {
+        Ok(runner) => runner,
+        Err(error) => {
+            return ScenarioResult {
+                name: candidate.name,
+                passed: false,
+                diagnostic: Some(error.to_string()),
+            };
+        }
+    };
+    if let Some(host_action_executor) = host_action_executor {
+        runner = runner.with_host_action_executor(host_action_executor);
+    }
+    eprintln!("CLI conformance run ID: {}", runner.id());
+    let scenario_result = match runner.check_gateway_status().await {
+        Ok(()) => candidate.run(&mut runner, plan_run).await,
+        Err(error) => Err(error),
+    };
+    let outcome = runner.finish(scenario_result).await;
+    ScenarioResult {
+        name: candidate.name,
+        passed: outcome.is_ok(),
+        diagnostic: outcome.err(),
+    }
+}
+
+async fn append_diagnostics(
+    mut result: ScenarioResult<'static>,
+    executor: &Arc<dyn HostActionExecutor>,
+    diagnostics: &openshell_conformance::PlanDiagnostics,
+) -> ScenarioResult<'static> {
+    let action = diagnostics.as_action();
+    if let Err(error) = executor.execute(&action).await {
+        let diagnostic = result.diagnostic.get_or_insert_default();
+        let _ = write!(diagnostic, "\n\nsecondary diagnostics failure:\n{error}");
     }
+    result
+}
 
+fn render_results(
+    results: Vec<ScenarioResult<'static>>,
+    output: OutputFormat,
+) -> Result<(), String> {
     let passed = results.iter().all(|result| result.passed);
     match output {
         OutputFormat::Text => {
@@ -178,20 +262,86 @@ async fn run(
     }
 }
 
+fn read_plan(path: &PathBuf) -> Result<ConformancePlan, String> {
+    let contents = if path.as_os_str() == "-" {
+        let mut input = String::new();
+        std::io::stdin()
+            .read_to_string(&mut input)
+            .map_err(|error| format!("read conformance plan from stdin: {error}"))?;
+        input
+    } else {
+        std::fs::read_to_string(path)
+            .map_err(|error| format!("read conformance plan {}: {error}", path.display()))?
+    };
+    ConformancePlan::parse(&contents).map_err(|error| format!("invalid conformance plan: {error}"))
+}
+
 fn select_scenarios(requested: &[String]) -> Result<Vec<&'static Scenario>, String> {
     if requested.is_empty() {
-        return Ok(scenarios().iter().collect());
+        return Ok(default_scenarios().collect());
     }
     requested
         .iter()
         .map(|name| {
-            scenario(name).ok_or_else(|| {
+            let candidate = scenario(name).ok_or_else(|| {
                 format!("unknown scenario '{name}'; run `openshell-conformance list`")
-            })
+            })?;
+            if candidate.requires_plan() {
+                return Err(format!(
+                    "scenario '{name}' requires an explicit --plan; run `openshell-conformance list`"
+                ));
+            }
+            Ok(candidate)
         })
         .collect()
 }
 
+struct ProcessHostAction;
+
+impl HostActionExecutor for ProcessHostAction {
+    fn execute(
+        &self,
+        action: &HostAction,
+    ) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send + '_>> {
+        let name = action.name.clone();
+        let command = action.command.clone();
+        let timeout = action.timeout();
+        let timeout_secs = action.timeout_secs;
+        Box::pin(async move {
+            let mut process = tokio::process::Command::new(&command);
+            process.kill_on_drop(true);
+            let output = tokio::time::timeout(timeout, process.output())
+                .await
+                .map_err(|_| {
+                    format!(
+                        "host action {:?} command '{}' timed out after {}s",
+                        name,
+                        command.display(),
+                        timeout_secs,
+                    )
+                })?
+                .map_err(|error| {
+                    format!(
+                        "start host action {:?} command '{}': {error}",
+                        name,
+                        command.display(),
+                    )
+                })?;
+            if output.status.success() {
+                return Ok(());
+            }
+            Err(format!(
+                "host action {:?} command '{}' exited {:?}:\nstdout:\n{}\nstderr:\n{}",
+                name,
+                command.display(),
+                output.status.code(),
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr),
+            ))
+        })
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use clap::Parser;
@@ -202,7 +352,7 @@ mod tests {
     fn selects_all_scenarios_by_default() {
         assert_eq!(
             select_scenarios(&[]).expect("select all").len(),
-            scenarios().len()
+            default_scenarios().count()
         );
     }
 
@@ -218,6 +368,14 @@ mod tests {
         assert!(error.contains("openshell-conformance list"));
     }
 
+    #[test]
+    fn action_scenario_requires_an_explicit_plan() {
+        let error = select_scenarios(&["sandbox-continuity".to_string()])
+            .expect_err("action scenario requires a plan");
+
+        assert!(error.contains("requires an explicit --plan"));
+    }
+
     #[test]
     fn parses_binary_override_and_json_output() {
         let cli = Cli::try_parse_from([
@@ -241,4 +399,14 @@ mod tests {
         assert_eq!(openshell_bin, Some(PathBuf::from("/opt/openshell")));
         assert_eq!(output, OutputFormat::Json);
     }
+
+    #[test]
+    fn parses_plan_from_stdin() {
+        let cli = Cli::try_parse_from(["openshell-conformance", "run", "--plan", "-"])
+            .expect("parse plan from stdin");
+        let Command::Run { plan, .. } = cli.command else {
+            panic!("expected run")
+        };
+        assert_eq!(plan, Some(PathBuf::from("-")));
+    }
 }