diff --git a/crates/openshell-core/src/backend_startup.rs b/crates/openshell-core/src/backend_startup.rs index 5c1769b39f..dff258834a 100644 --- a/crates/openshell-core/src/backend_startup.rs +++ b/crates/openshell-core/src/backend_startup.rs @@ -1,12 +1,94 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Closed startup failure codes carried by pre-start configuration refusals. +//! Closed diagnostics for authenticated startup reports and trusted launcher exits. //! //! The supervisor reports a code in the existing admission error field. The //! gateway projects only these fixed diagnostics, never backend error text. -/// The trusted operation that failed before workload execution. +/// Reserved exit of a trusted supervisor whose workload launch did not +/// complete unambiguously. It does not prove workload code never ran. +pub const SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED: i32 = 84; + +/// Public condition reason for an ambiguous workload launch. +pub const LAUNCH_UNCONFIRMED_CONDITION: &str = "WorkloadLaunchUnconfirmed"; + +/// Fixed diagnostic for the trusted supervisor's ambiguous launch failure. +pub const LAUNCH_UNCONFIRMED_DIAGNOSTIC: &str = "Workload startup failed or could not be confirmed; check the executable, working directory, execution permissions and runtime, then stop the sandbox and wait for cleanup before retrying"; + +/// A mandatory check failed in the trusted runtime launcher, before it +/// consumed the protected bootstrap or started any workload process. +/// +/// Drivers may interpret these codes only for the runtime launcher itself, +/// never for the canonical workload's separately reported exit status. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeStartupFailure { + /// The host cannot provide the mandatory Landlock ABI. + LandlockUnavailable, + /// The active Landlock allow/deny check failed. + LandlockEnforcement, + /// Another mandatory runtime isolation check failed. + Qualification, +} + +impl RuntimeStartupFailure { + /// Exit status owned by the trusted launcher, not by an agent process. + #[must_use] + pub const fn exit_code(self) -> u8 { + match self { + Self::LandlockUnavailable => 81, + Self::LandlockEnforcement => 82, + Self::Qualification => 83, + } + } + + /// Classify an observed exit of the trusted runtime launcher. + #[must_use] + pub const fn from_exit_code(code: i64) -> Option { + match code { + 81 => Some(Self::LandlockUnavailable), + 82 => Some(Self::LandlockEnforcement), + 83 => Some(Self::Qualification), + _ => None, + } + } + + /// Fixed condition reason for an observed trusted-launcher failure. + #[must_use] + pub const fn condition_reason(self) -> &'static str { + match self { + Self::LandlockUnavailable => "LandlockUnavailable", + Self::LandlockEnforcement => "LandlockEnforcementFailed", + Self::Qualification => "RuntimeQualificationFailed", + } + } + + /// Fixed public diagnostic without paths, arguments, or probe output. + #[must_use] + pub const fn diagnostic(self) -> &'static str { + match self { + Self::LandlockUnavailable => { + "Sandbox runtime qualification failed: Landlock ABI v3 or newer is unavailable; check the kernel used by the runtime and its seccomp configuration" + } + Self::LandlockEnforcement => { + "Sandbox runtime qualification failed: the Landlock allow/deny check failed; check the host filesystem and runtime configuration" + } + Self::Qualification => { + "Sandbox runtime qualification failed: a mandatory isolation check failed; check host prerequisites and runtime configuration" + } + } + } +} + +impl std::fmt::Display for RuntimeStartupFailure { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.diagnostic()) + } +} + +impl std::error::Error for RuntimeStartupFailure {} + +/// The trusted operation whose startup outcome failed. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BackendStartupStage { /// Constructing the selected backend client. @@ -15,6 +97,8 @@ pub enum BackendStartupStage { Attach, /// Confirming the isolation boundary before starting workload code. Confirm, + /// Launching the workload entrypoint after isolation confirmation. + StartAgent, } impl BackendStartupStage { @@ -25,6 +109,7 @@ impl BackendStartupStage { Self::Build => "build", Self::Attach => "attach", Self::Confirm => "confirm", + Self::StartAgent => "start-agent", } } } @@ -32,11 +117,14 @@ impl BackendStartupStage { /// Project an exact startup failure code into a bounded, credential-free /// diagnostic. Unknown and legacy error strings have no projection. /// -/// Only use this projection for an authenticated, generation-fenced -/// `refused_before_start` report; ordinary runtime errors are not startup claims. +/// Only use this projection for an authenticated, generation-fenced startup +/// report. An unconfirmed launch does not prove that workload execution stopped. #[must_use] pub fn backend_startup_diagnostic(code: &str) -> Option { let (stage, kind) = code.strip_prefix("openshell.backend.")?.split_once('.')?; + if stage == "start-agent" { + return (kind == "unconfirmed").then(|| LAUNCH_UNCONFIRMED_DIAGNOSTIC.to_string()); + } let stage = match stage { "build" => "client construction", "attach" => "attachment", diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 6b761f2f59..d971cc2d3d 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -23,6 +23,10 @@ use bollard::query_parameters::{ }; use bytes::Bytes; use futures::{Stream, StreamExt}; +use openshell_core::backend_startup::{ + LAUNCH_UNCONFIRMED_CONDITION, LAUNCH_UNCONFIRMED_DIAGNOSTIC, RuntimeStartupFailure, + SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED, +}; use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ @@ -5653,7 +5657,7 @@ async fn wait_for_docker_supervisor_ready( .await { Ok(()) => Ok(()), - Err(status) => Err(recorded_workspace_validation_failure(failures, sandbox_id) + Err(status) => Err(recorded_supervisor_start_failure(failures, sandbox_id) .await .unwrap_or(status)), } @@ -5675,7 +5679,14 @@ async fn poll_docker_supervisor_ready( .map_err(|error| { Status::internal(format!("inspect Docker sandbox container: {error}")) })?; - if sandbox.state.unwrap_or_default().running == Some(false) { + let workload_state = sandbox.state.unwrap_or_default(); + if workload_state.running == Some(false) { + if let Some(failure) = workload_state + .exit_code + .and_then(RuntimeStartupFailure::from_exit_code) + { + return Err(Status::failed_precondition(failure.diagnostic())); + } return Err(Status::unavailable( "Docker sandbox exited before supervisor became ready", )); @@ -5697,6 +5708,11 @@ async fn poll_docker_supervisor_ready( warn!(sandbox_id, supervisor_id, supervisor_logs = %log_tail, "Docker sandbox rejected the image working directory"); return Err(workspace_validation_status()); } + _ if state.running == Some(false) + && state.exit_code == Some(i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED)) => + { + return Err(Status::unavailable(LAUNCH_UNCONFIRMED_DIAGNOSTIC)); + } _ if state.running == Some(false) => { let log_tail = docker_container_log_tail(docker, supervisor_id).await; warn!(sandbox_id, supervisor_id, supervisor_logs = %log_tail, "Docker supervisor exited before becoming ready"); @@ -5716,6 +5732,20 @@ fn docker_supervisor_wait_failure( result: Option>, ) -> (&'static str, String) { match result { + Some(Ok(status)) if status.status_code == i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED) => { + ( + LAUNCH_UNCONFIRMED_CONDITION, + LAUNCH_UNCONFIRMED_DIAGNOSTIC.to_string(), + ) + } + Some(Err(BollardError::DockerContainerWaitError { code, .. })) + if code == i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED) => + { + ( + LAUNCH_UNCONFIRMED_CONDITION, + LAUNCH_UNCONFIRMED_DIAGNOSTIC.to_string(), + ) + } Some(Ok(status)) => ( supervisor_exit_reason(status.status_code), format!( @@ -5738,15 +5768,20 @@ fn docker_supervisor_wait_failure( } } -async fn recorded_workspace_validation_failure( +async fn recorded_supervisor_start_failure( failures: &Arc>>, sandbox_id: &str, ) -> Option { let failures = failures.lock().await; failures .get(sandbox_id) - .filter(|failure| failure.reason == CONDITION_WORKSPACE_VALIDATION_FAILED) - .map(|_| workspace_validation_status()) + .and_then(|failure| match failure.reason { + CONDITION_WORKSPACE_VALIDATION_FAILED => Some(workspace_validation_status()), + LAUNCH_UNCONFIRMED_CONDITION => { + Some(Status::unavailable(LAUNCH_UNCONFIRMED_DIAGNOSTIC)) + } + _ => None, + }) } /// Startup failure for a supervisor that exited because the sandbox rejected @@ -5763,8 +5798,22 @@ fn supervisor_start_failure_reason(status: &Status, default: &'static str) -> &' && status.message() == WORKSPACE_VALIDATION_FAILED_MESSAGE { CONDITION_WORKSPACE_VALIDATION_FAILED + } else if status.code() == tonic::Code::Unavailable + && status.message() == LAUNCH_UNCONFIRMED_DIAGNOSTIC + { + LAUNCH_UNCONFIRMED_CONDITION } else { - default + [ + RuntimeStartupFailure::LandlockUnavailable, + RuntimeStartupFailure::LandlockEnforcement, + RuntimeStartupFailure::Qualification, + ] + .into_iter() + .find(|failure| { + status.code() == tonic::Code::FailedPrecondition + && status.message() == failure.diagnostic() + }) + .map_or(default, RuntimeStartupFailure::condition_reason) } } @@ -5772,6 +5821,8 @@ fn supervisor_start_failure_reason(status: &Status, default: &'static str) -> &' fn supervisor_exit_reason(status_code: i64) -> &'static str { if status_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) { CONDITION_WORKSPACE_VALIDATION_FAILED + } else if status_code == i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED) { + LAUNCH_UNCONFIRMED_CONDITION } else { "ControlSupervisorExited" } @@ -6576,7 +6627,7 @@ fn apply_docker_exit_classification(sandbox: &mut DriverSandbox, state: &Contain if state.oom_killed == Some(true) { return; } - let Some(code) = state.exit_code.filter(|&code| matches!(code, 137 | 143)) else { + let Some(code) = state.exit_code else { return; }; let Some(condition) = sandbox @@ -6589,8 +6640,13 @@ fn apply_docker_exit_classification(sandbox: &mut DriverSandbox, state: &Contain if condition.reason != CONDITION_EXITED { return; } - condition.reason = CONDITION_RUNTIME_RESTART.to_string(); - condition.message = format!("Container terminated by signal (exit code {code})"); + if let Some(failure) = RuntimeStartupFailure::from_exit_code(code) { + condition.reason = failure.condition_reason().to_string(); + condition.message = failure.diagnostic().to_string(); + } else if matches!(code, 137 | 143) { + condition.reason = CONDITION_RUNTIME_RESTART.to_string(); + condition.message = format!("Container terminated by signal (exit code {code})"); + } } fn container_ready_condition( diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 92d20739f3..96d8a1f4a3 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -4378,6 +4378,29 @@ fn docker_oom_kill_stays_terminal_despite_137() { assert_eq!(ready_reason(&sandbox), CONDITION_EXITED); } +#[test] +fn runtime_qualification_exit_reports_fixed_failure() { + for failure in [ + RuntimeStartupFailure::LandlockUnavailable, + RuntimeStartupFailure::LandlockEnforcement, + RuntimeStartupFailure::Qualification, + ] { + let mut sandbox = exited_sandbox_with_ready_reason(CONDITION_EXITED); + let state = ContainerState { + status: Some(ContainerStateStatusEnum::EXITED), + exit_code: Some(i64::from(failure.exit_code())), + ..Default::default() + }; + apply_docker_exit_classification(&mut sandbox, &state); + assert_eq!(ready_reason(&sandbox), failure.condition_reason()); + let status = Status::failed_precondition(failure.diagnostic()); + assert_eq!( + supervisor_start_failure_reason(&status, "fallback"), + failure.condition_reason() + ); + } +} + #[test] fn supervisor_workspace_validation_exit_is_reported_explicitly() { let status = workspace_validation_status(); @@ -4398,6 +4421,35 @@ fn supervisor_workspace_validation_exit_is_reported_explicitly() { CONDITION_WORKSPACE_VALIDATION_FAILED ); assert_eq!(supervisor_exit_reason(1), "ControlSupervisorExited"); + assert_eq!( + supervisor_exit_reason(i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED)), + LAUNCH_UNCONFIRMED_CONDITION + ); + assert_eq!( + supervisor_start_failure_reason( + &Status::unavailable(LAUNCH_UNCONFIRMED_DIAGNOSTIC), + "fallback" + ), + LAUNCH_UNCONFIRMED_CONDITION + ); + for wait in [ + Some(Ok(ContainerWaitResponse { + status_code: i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED), + ..Default::default() + })), + Some(Err(BollardError::DockerContainerWaitError { + code: i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED), + error: "PRIVATE STDERR".into(), + })), + ] { + assert_eq!( + docker_supervisor_wait_failure(wait), + ( + LAUNCH_UNCONFIRMED_CONDITION, + LAUNCH_UNCONFIRMED_DIAGNOSTIC.to_string() + ) + ); + } let (reason, message) = docker_supervisor_wait_failure(Some(Err(BollardError::DockerContainerWaitError { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index afc06e58c2..5573522e5b 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -50,6 +50,10 @@ use kube::runtime::wait::await_condition; use kube::runtime::watcher::{self, Event}; use kube::{Client, Error as KubeError}; use openshell_core::NetworkSupervisorTrustBundle; +use openshell_core::backend_startup::{ + LAUNCH_UNCONFIRMED_CONDITION, LAUNCH_UNCONFIRMED_DIAGNOSTIC, RuntimeStartupFailure, + SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED, +}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ LABEL_GATEWAY_ID, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, @@ -4988,6 +4992,7 @@ impl KubernetesComputeDriver { &object, ) .await + .0 { SandboxRuntimeControlAvailability::Available | SandboxRuntimeControlAvailability::Degraded => {} @@ -5015,6 +5020,7 @@ impl KubernetesComputeDriver { &object, ) .await + .0 { SandboxRuntimeControlAvailability::Available | SandboxRuntimeControlAvailability::Degraded => {} @@ -6840,36 +6846,94 @@ fn sandbox_runtime_readiness_state( } } +fn runtime_startup_failure_from_pod(pod: &Pod) -> Option<(&'static str, &'static str)> { + let container = pod + .spec + .as_ref()? + .containers + .iter() + .find(|container| container.name == "agent")?; + let command = container.command.as_ref()?; + if command.first()? != &format!("{SANDBOX_RUNTIME_MOUNT_PATH}/openshell-sandbox") + || command.get(1).map(String::as_str) != Some("--bootstrap") + { + return None; + } + let terminated = pod + .status + .as_ref()? + .container_statuses + .as_ref()? + .iter() + .find(|container| container.name == "agent")? + .state + .as_ref()? + .terminated + .as_ref()?; + if terminated.signal.is_some_and(|signal| signal != 0) + || terminated.reason.as_deref() == Some("OOMKilled") + { + return None; + } + let failure = RuntimeStartupFailure::from_exit_code(i64::from(terminated.exit_code))?; + Some((failure.condition_reason(), failure.diagnostic())) +} + +fn supervisor_startup_failure_from_pod(pod: &Pod) -> Option<(&'static str, &'static str)> { + let terminated = pod + .status + .as_ref()? + .container_statuses + .as_ref()? + .iter() + .find(|container| container.name == "supervisor")? + .state + .as_ref()? + .terminated + .as_ref()?; + if terminated.signal.is_some_and(|signal| signal != 0) + || terminated.reason.as_deref() == Some("OOMKilled") + { + return None; + } + (terminated.exit_code == SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED) + .then_some((LAUNCH_UNCONFIRMED_CONDITION, LAUNCH_UNCONFIRMED_DIAGNOSTIC)) +} + async fn sandbox_runtime_workload_generation_matches( client: &Client, namespace: &str, pod_name: &str, sandbox: &DynamicObject, -) -> SandboxRuntimeControlAvailability { +) -> ( + SandboxRuntimeControlAvailability, + Option<(&'static str, &'static str)>, +) { let Some(expected_uid) = sandbox .metadata .annotations .as_ref() .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID)) else { - return SandboxRuntimeControlAvailability::Unavailable; + return (SandboxRuntimeControlAvailability::Unavailable, None); }; let pods = Api::::namespaced(client.clone(), namespace); match tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(pod_name)).await { - Ok(Ok(Some(pod))) if pod.metadata.uid.as_deref() == Some(expected_uid.as_str()) => { - SandboxRuntimeControlAvailability::Available - } - Ok(Ok(_)) => SandboxRuntimeControlAvailability::Unavailable, + Ok(Ok(Some(pod))) if pod.metadata.uid.as_deref() == Some(expected_uid.as_str()) => ( + SandboxRuntimeControlAvailability::Available, + runtime_startup_failure_from_pod(&pod), + ), + Ok(Ok(_)) => (SandboxRuntimeControlAvailability::Unavailable, None), Ok(Err(error)) => { warn!(pod = pod_name, %error, "could not verify sandbox-runtime workload generation"); - SandboxRuntimeControlAvailability::Unknown + (SandboxRuntimeControlAvailability::Unknown, None) } Err(_) => { warn!( pod = pod_name, "timed out checking sandbox-runtime workload generation" ); - SandboxRuntimeControlAvailability::Unknown + (SandboxRuntimeControlAvailability::Unknown, None) } } } @@ -6879,14 +6943,17 @@ async fn sandbox_runtime_supervisor_generation_matches( namespace: &str, sandbox_id: &str, sandbox: &DynamicObject, -) -> SandboxRuntimeControlAvailability { +) -> ( + SandboxRuntimeControlAvailability, + Option<(&'static str, &'static str)>, +) { match supervisor_placement(sandbox) { Ok(SupervisorPlacement::Dedicated) => {} // The sandbox instance's sandbox is checked with its availability. Ok(SupervisorPlacement::Shared { .. }) => { - return SandboxRuntimeControlAvailability::Available; + return (SandboxRuntimeControlAvailability::Available, None); } - Err(_) => return SandboxRuntimeControlAvailability::Unavailable, + Err(_) => return (SandboxRuntimeControlAvailability::Unavailable, None), } let Some(expected_uid) = sandbox .metadata @@ -6894,25 +6961,26 @@ async fn sandbox_runtime_supervisor_generation_matches( .as_ref() .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID)) else { - return SandboxRuntimeControlAvailability::Unavailable; + return (SandboxRuntimeControlAvailability::Unavailable, None); }; let names = SandboxRuntimeNames::new(sandbox_id); let pods = Api::::namespaced(client.clone(), namespace); match tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(&names.supervisor_pod)).await { - Ok(Ok(Some(pod))) if pod.metadata.uid.as_deref() == Some(expected_uid.as_str()) => { - SandboxRuntimeControlAvailability::Available - } - Ok(Ok(_)) => SandboxRuntimeControlAvailability::Unavailable, + Ok(Ok(Some(pod))) if pod.metadata.uid.as_deref() == Some(expected_uid.as_str()) => ( + SandboxRuntimeControlAvailability::Available, + supervisor_startup_failure_from_pod(&pod), + ), + Ok(Ok(_)) => (SandboxRuntimeControlAvailability::Unavailable, None), Ok(Err(error)) => { warn!(pod = names.supervisor_pod, %error, "could not verify sandbox-runtime supervisor generation"); - SandboxRuntimeControlAvailability::Unknown + (SandboxRuntimeControlAvailability::Unknown, None) } Err(_) => { warn!( pod = names.supervisor_pod, "timed out checking sandbox-runtime supervisor generation" ); - SandboxRuntimeControlAvailability::Unknown + (SandboxRuntimeControlAvailability::Unknown, None) } } } @@ -6954,8 +7022,11 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness( &sandbox_id, &obj, )); - let (dependencies, workload_generation, supervisor_generation) = - tokio::join!(dependencies, workload_generation, supervisor_generation); + let ( + dependencies, + (workload_generation, workload_failure), + (supervisor_generation, supervisor_failure), + ) = tokio::join!(dependencies, workload_generation, supervisor_generation); // A transient API read failure is not evidence that the running // boundary disappeared. Preserve the CR's published readiness for // Unknown and let periodic reconciliation retry. @@ -6968,6 +7039,21 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness( { mark_sandbox_runtime_control_unavailable(&mut sandbox); } + if sandbox_runtime_should_run(&obj) + && let Some((reason, message)) = workload_failure.or(supervisor_failure) + { + let status = sandbox.status.get_or_insert_with(Default::default); + status.conditions.retain(|condition| { + condition.r#type != "Ready" && condition.r#type != SANDBOX_SUSPENDED_CONDITION + }); + status.conditions.push(SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: reason.to_string(), + message: message.to_string(), + transition_time: None, + }); + } } Ok((name, sandbox)) } @@ -14451,6 +14537,44 @@ mod tests { })); } + #[test] + fn trusted_pod_exit_reports_fixed_startup_failure() { + let mut pod: Pod = serde_json::from_value(serde_json::json!({ + "metadata": {"name": "runtime"}, + "spec": {"containers": [{"name": "agent", "command": [format!("{SANDBOX_RUNTIME_MOUNT_PATH}/openshell-sandbox"), "--bootstrap", BOUNDARY_CONFIG_PATH]}]}, + "status": {"containerStatuses": [{"name": "agent", "image": "test", "imageID": "test", "ready": false, "restartCount": 0, "state": {"terminated": {"exitCode": 81}}}]} + })).unwrap(); + assert_eq!( + runtime_startup_failure_from_pod(&pod), + Some(( + RuntimeStartupFailure::LandlockUnavailable.condition_reason(), + RuntimeStartupFailure::LandlockUnavailable.diagnostic() + )) + ); + pod.spec.as_mut().unwrap().containers[0].command = Some(vec!["untrusted-command".into()]); + assert_eq!(runtime_startup_failure_from_pod(&pod), None); + let state = &mut pod + .status + .as_mut() + .unwrap() + .container_statuses + .as_mut() + .unwrap()[0]; + state.name = "supervisor".into(); + state + .state + .as_mut() + .unwrap() + .terminated + .as_mut() + .unwrap() + .exit_code = SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED; + assert_eq!( + supervisor_startup_failure_from_pod(&pod), + Some((LAUNCH_UNCONFIRMED_CONDITION, LAUNCH_UNCONFIRMED_DIAGNOSTIC)) + ); + } + #[test] fn completed_sandbox_runtime_bootstrap_preserves_real_suspension() { let sandbox = Sandbox { diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index 9a508e52c3..66aeb053bd 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -12,6 +12,10 @@ use crate::container::{ }; use futures::Stream; use openshell_core::ComputeDriverError; +use openshell_core::backend_startup::{ + LAUNCH_UNCONFIRMED_CONDITION, LAUNCH_UNCONFIRMED_DIAGNOSTIC, RuntimeStartupFailure, + SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED, +}; use openshell_core::proto::compute::v1::{ DriverCondition, DriverSandbox, DriverSandboxStatus, WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesSandboxEvent, watch_sandboxes_event, @@ -570,6 +574,15 @@ fn condition_from_state(state: &ContainerState) -> DriverCondition { CONDITION_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_FAILED_MESSAGE.to_string(), ) + } else if state.supervisor_exit_code + == Some(i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED)) + { + ( + LAUNCH_UNCONFIRMED_CONDITION, + LAUNCH_UNCONFIRMED_DIAGNOSTIC.to_string(), + ) + } else if let Some(failure) = RuntimeStartupFailure::from_exit_code(state.exit_code) { + (failure.condition_reason(), failure.diagnostic().to_string()) } else if matches!(state.exit_code, 137 | 143) { ( CONDITION_RUNTIME_RESTART, @@ -930,6 +943,28 @@ mod tests { state.exit_code = i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED); state.supervisor_exit_code = Some(1); assert_eq!(condition_from_state(&state).reason, CONDITION_EXITED); + state.supervisor_exit_code = Some(i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED)); + assert_eq!( + condition_from_state(&state).reason, + LAUNCH_UNCONFIRMED_CONDITION + ); + assert_eq!( + condition_from_state(&state).message, + LAUNCH_UNCONFIRMED_DIAGNOSTIC + ); + state.supervisor_exit_code = None; + state.exit_code = i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED); + assert_eq!(condition_from_state(&state).reason, CONDITION_EXITED); + for failure in [ + RuntimeStartupFailure::LandlockUnavailable, + RuntimeStartupFailure::LandlockEnforcement, + RuntimeStartupFailure::Qualification, + ] { + state.exit_code = i64::from(failure.exit_code()); + let condition = condition_from_state(&state); + assert_eq!(condition.reason, failure.condition_reason()); + assert_eq!(condition.message, failure.diagnostic()); + } } #[test] diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index f7946c34c2..c4648c58f0 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -45,6 +45,9 @@ use oci_client::manifest::{ use oci_client::secrets::RegistryAuth; use oci_client::{Reference, RegistryOperation}; use openshell_core::UpstreamProxyConfig; +use openshell_core::backend_startup::{ + LAUNCH_UNCONFIRMED_CONDITION, LAUNCH_UNCONFIRMED_DIAGNOSTIC, SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED, +}; use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::gpu::{ @@ -5077,11 +5080,23 @@ impl VmDriver { let _ = terminate_vm_process(&mut process.child).await; } } - let mut message = status.code().map_or_else( - || format!("{component} process exited"), - |code| format!("{component} process exited with status {code}"), - ); + let launch_unconfirmed = component == "host supervisor" + && status.code() == Some(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED); + let reason = if launch_unconfirmed { + LAUNCH_UNCONFIRMED_CONDITION + } else { + "ProcessExited" + }; + let mut message = if launch_unconfirmed { + LAUNCH_UNCONFIRMED_DIAGNOSTIC.to_string() + } else { + status.code().map_or_else( + || format!("{component} process exited"), + |code| format!("{component} process exited with status {code}"), + ) + }; if component == "host supervisor" + && !launch_unconfirmed && let Some(state_dir) = state_dir.as_deref() && let Some(stderr) = read_vm_console_tail( &state_dir.join("supervisor.err.log"), @@ -5092,18 +5107,14 @@ impl VmDriver { .expect("writing to String cannot fail"); } if let Some(snapshot) = self - .set_snapshot_condition( - &sandbox_id, - error_condition("ProcessExited", &message), - false, - ) + .set_snapshot_condition(&sandbox_id, error_condition(reason, &message), false) .await { self.publish_snapshot(snapshot); } self.publish_platform_event( sandbox_id.clone(), - platform_event("vm", "Warning", "ProcessExited", message), + platform_event("vm", "Warning", reason, message), ); let (has_gpu, cleanup_ctx) = { let registry = self.registry.lock().await; @@ -9337,9 +9348,10 @@ mod tests { .spawn() .unwrap() }; - for (component, child) in [ - ("VM", spawn_exited_child()), - ("host supervisor", running_child()), + for (component, child, launch_unconfirmed) in [ + ("VM", spawn_exited_child(), false), + ("host supervisor", running_child(), false), + ("host supervisor", running_child(), true), ] { let temp = tempfile::tempdir().unwrap(); std::fs::write(temp.path().join("rootfs-console.log"), "agent output\n").unwrap(); @@ -9348,7 +9360,29 @@ mod tests { let mut events = driver.events.subscribe(); insert_test_record(&driver, "sb-exit", temp.path().to_path_buf(), child).await; + if launch_unconfirmed { + let process = driver.registry.lock().await["sb-exit"] + .process + .as_ref() + .unwrap() + .clone(); + let mut process = process.lock().await; + process.supervisor.wait().await.unwrap(); + process.supervisor = Command::new("sh") + .arg("-c") + .arg("exit 84") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + } driver.monitor_sandbox("sb-exit".to_string()).await; + let expected_reason = if launch_unconfirmed { + LAUNCH_UNCONFIRMED_CONDITION + } else { + "ProcessExited" + }; let condition = driver.registry.lock().await["sb-exit"] .snapshot @@ -9358,7 +9392,7 @@ mod tests { status .conditions .iter() - .find(|condition| condition.reason == "ProcessExited") + .find(|condition| condition.reason == expected_reason) .cloned() }) .expect("ProcessExited condition"); @@ -9366,17 +9400,22 @@ mod tests { while let Ok(event) = events.try_recv() { if let Some(watch_sandboxes_event::Payload::PlatformEvent(platform)) = event.payload && let Some(event) = platform.event - && event.reason == "ProcessExited" + && event.reason == expected_reason { event_message = Some(event.message); } } - let event_message = event_message.expect("ProcessExited platform event"); + let event_message = event_message.expect("process failure platform event"); for message in [&condition.message, &event_message] { - assert!(message.starts_with(&format!("{component} process exited"))); + if launch_unconfirmed { + assert_eq!(message.as_str(), LAUNCH_UNCONFIRMED_DIAGNOSTIC); + assert!(!message.contains("supervisor error")); + } else { + assert!(message.starts_with(&format!("{component} process exited"))); + } assert!(!message.contains("agent output"), "{component}: {message}"); } - if component == "host supervisor" { + if !launch_unconfirmed && component == "host supervisor" { assert!(condition.message.contains("supervisor error")); } } diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 86ef2bb4f5..315dcd5fb1 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -6,6 +6,7 @@ #[cfg(target_os = "linux")] use std::mem::size_of; use std::path::Path; +use std::process::ExitCode; use clap::Parser; use miette::{IntoDiagnostic, Result}; @@ -156,9 +157,10 @@ fn qualify_runtime() -> Result<(openshell_sandbox::RuntimeQualification, Qualifi let task_memory_copy = true; // Check the ABI first: on an older kernel the allow/deny probe would fail // with an opaque ruleset error instead of naming the ABI it needs. - let landlock_abi = - openshell_sandbox::sandbox::linux::require_landlock_abi().wrap_err("Landlock ABI probe")?; - probe_landlock_allow_deny().wrap_err("Landlock allow/deny probe")?; + let landlock_abi = openshell_sandbox::sandbox::linux::require_landlock_abi() + .wrap_err(openshell_core::backend_startup::RuntimeStartupFailure::LandlockUnavailable)?; + probe_landlock_allow_deny() + .wrap_err(openshell_core::backend_startup::RuntimeStartupFailure::LandlockEnforcement)?; let notification = openshell_sandbox::linux::seccomp_notify::probe_notification_api() .into_diagnostic() .wrap_err("seccomp notification probe")?; @@ -1770,7 +1772,16 @@ fn run_boundary(bootstrap: &Path, log_level: &str) -> Result<()> { .with_filter(console_filter), ) .try_init(); - let (qualification, _) = qualify_runtime()?; + let (qualification, _) = qualify_runtime().map_err(|error| { + if error + .downcast_ref::() + .is_some() + { + error + } else { + error.wrap_err(openshell_core::backend_startup::RuntimeStartupFailure::Qualification) + } + })?; openshell_sandbox::run(bootstrap, qualification) } @@ -1779,7 +1790,27 @@ fn run_boundary(_bootstrap: &Path, _log_level: &str) -> Result<()> { Err(miette::miette!("openshell-sandbox requires Linux")) } -fn main() -> Result<()> { +fn main() -> Result { + runtime_exit(run()) +} + +fn runtime_exit(result: Result<()>) -> Result { + match result { + Ok(()) => Ok(ExitCode::SUCCESS), + Err(error) => { + if let Some(failure) = + error.downcast_ref::() + { + eprintln!("{error:?}"); + Ok(ExitCode::from(failure.exit_code())) + } else { + Err(error) + } + } + } +} + +fn run() -> Result<()> { let raw_args = std::env::args().collect::>(); if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) { let dest = raw_args.get(2).ok_or_else(|| { @@ -1961,3 +1992,26 @@ mod tests { assert!(final_path.exists(), "binary should land inside dest dir"); } } + +#[cfg(test)] +mod startup_exit_tests { + use super::*; + use openshell_core::backend_startup::RuntimeStartupFailure; + + #[test] + fn only_typed_qualification_failures_get_reserved_exit_statuses() { + for failure in [ + RuntimeStartupFailure::LandlockUnavailable, + RuntimeStartupFailure::LandlockEnforcement, + RuntimeStartupFailure::Qualification, + ] { + let error = miette::miette!("private probe detail").wrap_err(failure); + assert_eq!( + runtime_exit(Err(error)).unwrap(), + ExitCode::from(failure.exit_code()) + ); + } + assert!(runtime_exit(Err(miette::miette!("LandlockUnavailable: private output"))).is_err()); + assert_eq!(runtime_exit(Ok(())).unwrap(), ExitCode::SUCCESS); + } +} diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 384745ea33..e69680dff3 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -5069,7 +5069,7 @@ pub(super) async fn handle_report_sandbox_configuration( state: &Arc, request: Request, ) -> Result, Status> { - use openshell_core::proto::ConfigurationAdmissionState; + use openshell_core::proto::{ConfigurationAdmissionState, SandboxPhase}; let sandbox_id = request.get_ref().sandbox_id.clone(); crate::auth::guard::enforce_sandbox_scope(&request, &sandbox_id)?; let mut admission = request @@ -5086,6 +5086,14 @@ pub(super) async fn handle_report_sandbox_configuration( return Err(Status::invalid_argument("admission state is required")); } let refused_before_start = request.get_ref().refused_before_start; + let launch_unconfirmed = admission.error == "openshell.backend.start-agent.unconfirmed"; + if launch_unconfirmed + && (refused_before_start || reported != ConfigurationAdmissionState::Rejected) + { + return Err(Status::invalid_argument( + "an unconfirmed workload launch requires a rejected report without refused_before_start", + )); + } if refused_before_start && reported != ConfigurationAdmissionState::Rejected { return Err(Status::invalid_argument( "refused_before_start requires a rejected admission", @@ -5112,13 +5120,15 @@ pub(super) async fn handle_report_sandbox_configuration( .ok_or_else(|| Status::not_found("sandbox not found"))?; let sandbox_name = sandbox.object_name().to_string(); let workspace = sandbox.object_workspace().to_string(); - crate::compute::provisioning_deadline::refresh_configuration( - &state.store, - &mut sandbox, - current_time_ms(), - ) - .await - .map_err(Status::internal)?; + if !launch_unconfirmed { + crate::compute::provisioning_deadline::refresh_configuration( + &state.store, + &mut sandbox, + current_time_ms(), + ) + .await + .map_err(Status::internal)?; + } if crate::compute::provisioning_deadline::timed_out(&sandbox) { return Err(Status::failed_precondition( "provisioning deadline expired; explicitly start the sandbox after cleanup", @@ -5144,7 +5154,20 @@ pub(super) async fn handle_report_sandbox_configuration( "supervisor configuration instance has changed", )); } - if reported == ConfigurationAdmissionState::Accepted { + if launch_unconfirmed + && current.is_none_or(|current| { + current.state != i32::from(ConfigurationAdmissionState::Accepted) + || !same_generation(current, &admission) + }) + { + return Err(Status::aborted( + "unconfirmed workload launch does not match the accepted configuration", + )); + } + if launch_unconfirmed { + // The accepted launch can lag a desired policy/provider update. + admission = current.expect("accepted generation was checked").clone(); + } else if reported == ConfigurationAdmissionState::Accepted { let mut config_request = Request::new(GetSandboxConfigRequest { name: sandbox_name.clone(), workspace_scope: Some(openshell_core::proto::workspace_selector(workspace.clone())), @@ -5218,11 +5241,13 @@ pub(super) async fn handle_report_sandbox_configuration( .map_err(Status::internal)?; return Err(Status::failed_precondition("provisioning deadline expired")); } - crate::compute::provisioning_deadline::record_admission_start(record, now_ms) - .map_err(Status::internal)?; - if reported == ConfigurationAdmissionState::Rejected { - crate::compute::provisioning_deadline::record_rejection(record, now_ms) + if !launch_unconfirmed { + crate::compute::provisioning_deadline::record_admission_start(record, now_ms) .map_err(Status::internal)?; + if reported == ConfigurationAdmissionState::Rejected { + crate::compute::provisioning_deadline::record_rejection(record, now_ms) + .map_err(Status::internal)?; + } } } let updated = state @@ -5244,7 +5269,34 @@ pub(super) async fn handle_report_sandbox_configuration( .expect("status initialized") .configuration_activated = Some(true); } - crate::compute::apply_configuration_readiness(sandbox); + if launch_unconfirmed { + // Lost launch confirmation is a runtime failure, not policy refusal. + let status = sandbox.status.as_mut().expect("status initialized"); + if matches!( + SandboxPhase::try_from(status.phase), + Ok(SandboxPhase::Provisioning | SandboxPhase::Starting | SandboxPhase::Ready) + ) || (status.phase == i32::from(SandboxPhase::Error) + && status.exit_code.is_none()) + { + status.phase = i32::from(SandboxPhase::Error); + status + .conditions + .retain(|condition| condition.r#type != "Ready"); + status + .conditions + .push(openshell_core::proto::SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: openshell_core::backend_startup::LAUNCH_UNCONFIRMED_CONDITION + .to_string(), + message: openshell_core::backend_startup::LAUNCH_UNCONFIRMED_DIAGNOSTIC + .to_string(), + ..Default::default() + }); + } + } else { + crate::compute::apply_configuration_readiness(sandbox); + } }) .await .map_err(|error| { @@ -9158,6 +9210,238 @@ mod tests { assert!(unknown.message.contains("isolation backend refused")); } + #[tokio::test] + async fn unconfirmed_launch_keeps_accepted_policy_and_does_not_reopen_repair() { + use openshell_core::backend_startup::{ + LAUNCH_UNCONFIRMED_CONDITION, LAUNCH_UNCONFIRMED_DIAGNOSTIC, + }; + use openshell_core::proto::{ + ConfigurationAdmissionState as Admission, ReportSandboxConfigurationRequest, + SandboxConfigurationAdmission, SandboxPhase, + }; + + let state = test_server_state().await; + let sandbox_id = "sb-launch-unconfirmed"; + let mut sandbox = test_sandbox( + sandbox_id, + "launch-unconfirmed", + openshell_policy::restrictive_default_policy(), + Vec::new(), + ); + sandbox.set_phase(SandboxPhase::Provisioning.into()); + sandbox.status.as_mut().unwrap().provisioning = Some( + crate::compute::provisioning_deadline::new_record(current_time_ms()), + ); + state.store.put_message(&sandbox).await.unwrap(); + let instance_id = uuid::Uuid::new_v4().to_string(); + let report = |admission, refused_before_start| { + with_sandbox( + Request::new(ReportSandboxConfigurationRequest { + sandbox_id: sandbox_id.to_string(), + admission: Some(admission), + refused_before_start, + ..Default::default() + }), + sandbox_id, + ) + }; + handle_report_sandbox_configuration( + &state, + report( + SandboxConfigurationAdmission { + instance_id: instance_id.clone(), + state: Admission::Pending.into(), + ..Default::default() + }, + false, + ), + ) + .await + .unwrap(); + let config = handle_get_sandbox_config( + &state, + with_sandbox( + Request::new(GetSandboxConfigRequest { + name: "launch-unconfirmed".to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + }), + sandbox_id, + ), + ) + .await + .unwrap() + .into_inner(); + let accepted = SandboxConfigurationAdmission { + instance_id, + state: Admission::Accepted.into(), + policy_version: config.version, + policy_hash: config.policy_hash, + config_revision: config.config_revision, + provider_env_revision: config.provider_env_revision, + error: String::new(), + }; + handle_report_sandbox_configuration(&state, report(accepted.clone(), false)) + .await + .unwrap(); + let before = state + .store + .get_message::(sandbox_id) + .await + .unwrap() + .unwrap(); + let mut unconfirmed = accepted.clone(); + unconfirmed.state = Admission::Rejected.into(); + unconfirmed.error = "openshell.backend.start-agent.unconfirmed".to_string(); + let mut wrong_generation = unconfirmed.clone(); + wrong_generation.policy_hash.push('x'); + assert_eq!( + handle_report_sandbox_configuration(&state, report(wrong_generation, false)) + .await + .unwrap_err() + .code(), + Code::Aborted + ); + assert_eq!( + handle_report_sandbox_configuration(&state, report(unconfirmed.clone(), true)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + // A newer desired policy does not supersede the in-flight accepted launch. + let mut desired = before.spec.as_ref().unwrap().policy.clone().unwrap(); + desired.network_policies.insert( + "new-endpoint".to_string(), + NetworkPolicyRule { + endpoints: vec![NetworkEndpoint { + host: "api.example.com".to_string(), + port: 443, + ..Default::default() + }], + ..Default::default() + }, + ); + handle_update_config( + &state, + with_user(Request::new(UpdateConfigRequest { + sandbox: "launch-unconfirmed".to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + policy: Some(desired), + ..Default::default() + })), + ) + .await + .unwrap(); + let before = state + .store + .get_message::(sandbox_id) + .await + .unwrap() + .unwrap(); + handle_report_sandbox_configuration(&state, report(unconfirmed.clone(), false)) + .await + .unwrap(); + let mut saved = state + .store + .get_message::(sandbox_id) + .await + .unwrap() + .unwrap(); + let status = saved.status.as_ref().unwrap(); + assert_eq!(status.configuration_admission.as_ref(), Some(&accepted)); + assert_eq!(status.configuration_activated, Some(true)); + assert_eq!( + status.provisioning, + before.status.as_ref().unwrap().provisioning + ); + assert_eq!(saved.phase(), i32::from(SandboxPhase::Error)); + assert!(!crate::policy_store::permits_initial_static_policy_repair( + &saved + )); + let ready = status + .conditions + .iter() + .find(|condition| condition.r#type == "Ready") + .unwrap(); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, LAUNCH_UNCONFIRMED_CONDITION); + assert_eq!(ready.message, LAUNCH_UNCONFIRMED_DIAGNOSTIC); + assert!(status.conditions.iter().any( + |condition| condition.r#type == "ConfigurationReady" && condition.status == "True" + )); + + // The driver may publish its generic error before this report arrives. + saved + .status + .as_mut() + .unwrap() + .conditions + .iter_mut() + .find(|condition| condition.r#type == "Ready") + .unwrap() + .reason = "ContainerExited".to_string(); + state.store.put_message(&saved).await.unwrap(); + handle_report_sandbox_configuration(&state, report(unconfirmed.clone(), false)) + .await + .unwrap(); + saved = state + .store + .get_message::(sandbox_id) + .await + .unwrap() + .unwrap(); + assert!( + saved + .status + .as_ref() + .unwrap() + .conditions + .iter() + .any(|condition| condition.r#type == "Ready" + && condition.reason == LAUNCH_UNCONFIRMED_CONDITION) + ); + + // Known canonical outcomes win over a later ambiguous launch report. + for (phase, exit_code, reason) in [ + (SandboxPhase::Completed, 0, "MainProcessCompleted"), + (SandboxPhase::Error, 17, "MainProcessFailed"), + ] { + saved.set_phase(phase.into()); + let status = saved.status.as_mut().unwrap(); + status.exit_code = Some(exit_code); + status + .conditions + .retain(|condition| condition.r#type != "Ready"); + status + .conditions + .push(openshell_core::proto::SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: reason.to_string(), + ..Default::default() + }); + state.store.put_message(&saved).await.unwrap(); + handle_report_sandbox_configuration(&state, report(unconfirmed.clone(), false)) + .await + .unwrap(); + let terminal = state + .store + .get_message::(sandbox_id) + .await + .unwrap() + .unwrap(); + assert_eq!(terminal.phase(), i32::from(phase)); + assert!( + terminal + .status + .unwrap() + .conditions + .iter() + .any(|condition| condition.r#type == "Ready" && condition.reason == reason) + ); + } + } + #[test] fn configuration_diagnostic_is_bounded_and_removes_control_characters() { let diagnostic = bounded_configuration_diagnostic(&format!("rule\n{}", "é".repeat(1000))); diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 41c5ace138..f407e8f6fb 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -118,6 +118,7 @@ struct Observed { require_networking: AtomicBool, active: AtomicBool, attach_error: Mutex>, + start_error: Mutex>, starts: AtomicUsize, releases: AtomicUsize, suspendable: AtomicBool, @@ -378,6 +379,10 @@ impl PreparedBackend for TestLaunch { } observed.record("start"); observed.starts.fetch_add(1, Ordering::SeqCst); + let start_error = observed.start_error.lock().unwrap().take(); + if let Some(error) = start_error { + return Err(error); + } if observed.defer_restore.load(Ordering::SeqCst) { assert!(observed.events().contains(&"confirm")); observed.restores.fetch_add(1, Ordering::SeqCst); @@ -979,6 +984,25 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { .observed .deny_confirmation .store(false, Ordering::SeqCst); + *setup.observed.start_error.lock().unwrap() = Some(BackendError::Process( + "spawn entrypoint: Permission denied (secret argument)".into(), + )); + let error = startup(false) + .await + .expect_err("entrypoint launch failure must end startup"); + assert_eq!( + error.code().map(|code| code.to_string()).as_deref(), + Some("openshell.backend.start-agent.failed") + ); + assert!(error.to_string().contains("secret argument")); + assert!(!readiness.exists()); + assert!(!marker.exists()); + setup.observed.events.lock().unwrap().truncate(7); + setup.observed.starts.store(0, Ordering::SeqCst); + tokio::task::yield_now().await; + assert!(!setup.observed.active.load(Ordering::SeqCst)); + assert_eq!(setup.observed.releases.load(Ordering::SeqCst), 2); + let observe_and_shutdown = async { tokio::time::timeout(Duration::from_secs(30), async { loop { @@ -1013,7 +1037,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { ] ); tokio::task::yield_now().await; - assert_eq!(setup.observed.releases.load(Ordering::SeqCst), 2); + assert_eq!(setup.observed.releases.load(Ordering::SeqCst), 3); assert!(!setup.observed.active.load(Ordering::SeqCst)); tokio::task::yield_now().await; assert!(setup.observed.services.lock().unwrap().upgrade().is_none()); diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index e09fd07d7b..a3883d58d4 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -88,11 +88,14 @@ pub struct WorkspaceValidationFailed(String); /// The sandbox reports errors across the boundary as text, so a workspace /// rejection is recognized here, once, and carried as /// [`WorkspaceValidationFailed`] from then on. -pub fn start_agent_error(message: String) -> miette::Report { +pub fn start_agent_error( + error: openshell_isolation_interface::contract::BackendError, +) -> miette::Report { + let message = error.to_string(); if message.contains(openshell_core::driver_utils::WORKSPACE_VALIDATION_ERROR_CONTEXT) { miette::Report::new(WorkspaceValidationFailed(message)) } else { - miette::miette!(message) + backend_setup::startup_error(BackendStartupStage::StartAgent, error) } } @@ -1674,7 +1677,7 @@ pub async fn run_sandbox_instance( sandbox_id: poll_id, sandbox: poll_sandbox, opa_engine: poll_engine, - loaded_policy_origin, + loaded_policy_origin: loaded_policy_origin.clone(), vm_identity: vm_policy_identity, entrypoint_pid: poll_pid, interval_secs: poll_interval_secs, @@ -1730,12 +1733,28 @@ pub async fn run_sandbox_instance( let (confirmed, backend_name) = remote_ready; let exit_code = { - let running = confirmed - .into_boundary() - .start_agent() - .instrument(tracing::info_span!(parent: &startup, "supervisor.boundary.start_agent")) - .await - .map_err(|error| start_agent_error(error.to_string()))?; + let started = + Box::pin(confirmed.into_boundary().start_agent().instrument( + tracing::info_span!(parent: &startup, "supervisor.boundary.start_agent"), + )) + .await; + let running = match started { + Ok(running) => running, + Err(error) => { + let error = start_agent_error(error); + if let (Some(sandbox_id), Some(gateway)) = (sandbox_id.as_deref(), gateway.as_ref()) + { + Box::pin(report_backend_startup_failure( + gateway, + sandbox_id, + &loaded_policy_origin, + &error, + )) + .await; + } + return Err(error); + } + }; // Opening can only prepare a restore. Report success after StartAgent // has completed restoration, so a late failure is never logged as resumed. if let Some(outcome) = running.launch_outcome() @@ -4229,7 +4248,7 @@ fn enqueue_policy_status(sender: &UnboundedSender, update: P } } -/// Report a classified backend failure before the workload has started. +/// Build an authenticated generation-bound startup failure report. fn backend_startup_failure_report( sandbox_id: &str, origin: &LoadedPolicyOrigin, @@ -4242,7 +4261,13 @@ fn backend_startup_failure_report( else { return None; }; - let code = error.code()?.to_string(); + let mut code = error.code()?.to_string(); + // Starting is a mutation: a lost reply can follow successful execution. + // This path cannot prove that workload execution did not begin. + let refused_before_start = !code.starts_with("openshell.backend.start-agent."); + if !refused_before_start { + code = "openshell.backend.start-agent.unconfirmed".to_string(); + } backend_startup_diagnostic(&code)?; Some(openshell_core::proto::ReportSandboxConfigurationRequest { sandbox_id: sandbox_id.to_string(), @@ -4257,11 +4282,11 @@ fn backend_startup_failure_report( error: code, }), expected_instance_id: String::new(), - refused_before_start: true, + refused_before_start, }) } -/// Report a pre-start failure without replacing the original error. The entire +/// Report a startup failure without replacing the original error. The entire /// best-effort operation, including any policy-history report, has one deadline. async fn report_backend_startup_failure( gateway: &openshell_core::grpc_client::GatewayEndpoint, @@ -6800,6 +6825,21 @@ network_policies: } let transient = miette::miette!("unavailable"); + for failure in [ + BackendError::Process("launch outcome is unknown".into()), + BackendError::Unavailable("reply was lost".into()), + BackendError::Denied("already-running inputs differ".into()), + BackendError::Terminated("runtime connection was lost".into()), + ] { + let error = start_agent_error(failure); + let request = backend_startup_failure_report("sandbox-id", &origin(&revision), &error) + .expect("an uncertain launch has a bounded infrastructure report"); + assert!(!request.refused_before_start); + assert_eq!( + request.admission.unwrap().error, + "openshell.backend.start-agent.unconfirmed" + ); + } assert!( backend_startup_failure_report("sandbox-id", &origin(&revision), &transient).is_none() ); @@ -6848,6 +6888,7 @@ network_policies: BackendStartupStage::Build, BackendStartupStage::Attach, BackendStartupStage::Confirm, + BackendStartupStage::StartAgent, ] { for failure in [ BackendError::Denied("policy rejected".into()), diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index f45c9f4f38..fdccc06153 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -850,15 +850,18 @@ fn otlp_span_filter(log_level: &str) -> EnvFilter { /// Choose the supervisor's exit status for a sandbox run. /// -/// The reserved status tells the compute driver that the sandbox rejected the -/// image working directory. After the workload exits, the supervisor exits -/// with the workload's status, so a workload that itself exits with the +/// Reserved statuses identify a rejected image working directory or a launch +/// whose outcome could not be confirmed. After the workload exits, the supervisor +/// exits with the workload's status, so a workload that itself exits with the /// reserved status is reported as 1 instead. The gateway receives the /// workload's exact status separately. fn supervisor_exit(result: Result) -> Result { + use openshell_core::backend_startup::SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED; use openshell_core::driver_utils::SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED; match result { - Ok(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) => Ok(1), + Ok(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED | SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED) => { + Ok(1) + } Err(error) if error .downcast_ref::() @@ -867,6 +870,15 @@ fn supervisor_exit(result: Result) -> Result { error!(error = ?error, "Image workspace validation failed"); Ok(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) } + Err(error) + if error.code().is_some_and(|code| { + code.to_string() + .starts_with("openshell.backend.start-agent.") + }) => + { + error!(error = ?error, "Workload launch did not complete"); + Ok(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED) + } other => other, } } @@ -960,7 +972,7 @@ mod tests { } #[test] - fn only_workspace_validation_failure_exits_with_reserved_status() { + fn startup_errors_use_reserved_statuses_without_relabeling_workload_exits() { use miette::WrapErr as _; use openshell_core::driver_utils::{ SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED, WORKSPACE_VALIDATION_ERROR_CONTEXT, @@ -972,18 +984,23 @@ mod tests { )) .wrap_err(WORKSPACE_VALIDATION_ERROR_CONTEXT) .unwrap_err(); - let error = openshell_supervisor::start_agent_error(format!( - "process error: boundary process leaf: start process supervisor leaf: {sandbox_error:?}" - )); + let error = openshell_supervisor::start_agent_error( + openshell_isolation_interface::contract::BackendError::Process(format!( + "boundary process leaf: start process supervisor leaf: {sandbox_error:?}" + )), + ); assert_eq!( supervisor_exit(Err(error)).expect("reserved exit status"), SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED ); - assert!( + assert_eq!( supervisor_exit(Err(openshell_supervisor::start_agent_error( - "connect failed".into() + openshell_isolation_interface::contract::BackendError::Process( + "connect failed".into() + ) ))) - .is_err() + .unwrap(), + openshell_core::backend_startup::SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED ); // Only an agent start failure carries the rejection, even when another // error's text contains the same words. @@ -1000,6 +1017,13 @@ mod tests { 1 ); assert_eq!(supervisor_exit(Ok(0)).unwrap(), 0); + assert_eq!( + supervisor_exit(Ok( + openshell_core::backend_startup::SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED + )) + .unwrap(), + 1 + ); assert_eq!(supervisor_exit(Ok(143)).unwrap(), 143); } diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 2ac6a4f69d..39e033b357 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -737,6 +737,19 @@ Print only the policy YAML for scripting (same effective policy, no metadata): openshell sandbox get my-sandbox --policy-only ``` +### Diagnose Workload Startup Failures + +If creation fails, use `openshell sandbox get my-sandbox` to inspect the startup +diagnostic. An entrypoint launch failure is different from a command that started +and exited with a nonzero status. Check that the configured executable and +working directory exist and that the executable has permission to run. + +Runtime qualification can also fail before the command starts. If Landlock is +unavailable, check the kernel and security configuration used by the selected +runtime; changing the workload's filesystem policy does not add kernel support. +If OpenShell cannot confirm whether the command started, stop the sandbox and +wait for cleanup before starting it again. + ### Diagnose Failed Calls to Tool Servers A sandbox can be `Ready` while an agent's call to an external tool server fails with `fetch failed`. Run `openshell sandbox get my-sandbox` and look under `Tool server connections` for the server's address, `Last result`, and `Reported at`. This is available for configured MCP-over-HTTP endpoints and does not change sandbox readiness.