fix(startup)!: report runtime and workload startup failures

crates/openshell-core/src/backend_startup.rs

@@ -1,12 +1,94 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-//! Closed startup failure codes carried by pre-start configuration refusals.
+//! Closed diagnostics for authenticated startup reports and trusted launcher exits.
 //!
 //! The supervisor reports a code in the existing admission error field. The
 //! gateway projects only these fixed diagnostics, never backend error text.
 
-/// The trusted operation that failed before workload execution.
+/// Reserved exit of a trusted supervisor whose workload launch did not
+/// complete unambiguously. It does not prove workload code never ran.
+pub const SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED: i32 = 84;
+
+/// Public condition reason for an ambiguous workload launch.
+pub const LAUNCH_UNCONFIRMED_CONDITION: &str = "WorkloadLaunchUnconfirmed";
+
+/// Fixed diagnostic for the trusted supervisor's ambiguous launch failure.
+pub const LAUNCH_UNCONFIRMED_DIAGNOSTIC: &str = "Workload startup failed or could not be confirmed; check the executable, working directory, execution permissions and runtime, then stop the sandbox and wait for cleanup before retrying";
+
+/// A mandatory check failed in the trusted runtime launcher, before it
+/// consumed the protected bootstrap or started any workload process.
+///
+/// Drivers may interpret these codes only for the runtime launcher itself,
+/// never for the canonical workload's separately reported exit status.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RuntimeStartupFailure {
+    /// The host cannot provide the mandatory Landlock ABI.
+    LandlockUnavailable,
+    /// The active Landlock allow/deny check failed.
+    LandlockEnforcement,
+    /// Another mandatory runtime isolation check failed.
+    Qualification,
+}
+
+impl RuntimeStartupFailure {
+    /// Exit status owned by the trusted launcher, not by an agent process.
+    #[must_use]
+    pub const fn exit_code(self) -> u8 {
+        match self {
+            Self::LandlockUnavailable => 81,
+            Self::LandlockEnforcement => 82,
+            Self::Qualification => 83,
+        }
+    }
+
+    /// Classify an observed exit of the trusted runtime launcher.
+    #[must_use]
+    pub const fn from_exit_code(code: i64) -> Option<Self> {
+        match code {
+            81 => Some(Self::LandlockUnavailable),
+            82 => Some(Self::LandlockEnforcement),
+            83 => Some(Self::Qualification),
+            _ => None,
+        }
+    }
+
+    /// Fixed condition reason for an observed trusted-launcher failure.
+    #[must_use]
+    pub const fn condition_reason(self) -> &'static str {
+        match self {
+            Self::LandlockUnavailable => "LandlockUnavailable",
+            Self::LandlockEnforcement => "LandlockEnforcementFailed",
+            Self::Qualification => "RuntimeQualificationFailed",
+        }
+    }
+
+    /// Fixed public diagnostic without paths, arguments, or probe output.
+    #[must_use]
+    pub const fn diagnostic(self) -> &'static str {
+        match self {
+            Self::LandlockUnavailable => {
+                "Sandbox runtime qualification failed: Landlock ABI v3 or newer is unavailable; check the kernel used by the runtime and its seccomp configuration"
+            }
+            Self::LandlockEnforcement => {
+                "Sandbox runtime qualification failed: the Landlock allow/deny check failed; check the host filesystem and runtime configuration"
+            }
+            Self::Qualification => {
+                "Sandbox runtime qualification failed: a mandatory isolation check failed; check host prerequisites and runtime configuration"
+            }
+        }
+    }
+}
+
+impl std::fmt::Display for RuntimeStartupFailure {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(self.diagnostic())
+    }
+}
+
+impl std::error::Error for RuntimeStartupFailure {}
+
+/// The trusted operation whose startup outcome failed.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum BackendStartupStage {
     /// Constructing the selected backend client.
@@ -15,6 +97,8 @@ pub enum BackendStartupStage {
     Attach,
     /// Confirming the isolation boundary before starting workload code.
     Confirm,
+    /// Launching the workload entrypoint after isolation confirmation.
+    StartAgent,
 }
 
 impl BackendStartupStage {
@@ -25,18 +109,22 @@ impl BackendStartupStage {
             Self::Build => "build",
             Self::Attach => "attach",
             Self::Confirm => "confirm",
+            Self::StartAgent => "start-agent",
         }
     }
 }
 
 /// Project an exact startup failure code into a bounded, credential-free
 /// diagnostic. Unknown and legacy error strings have no projection.
 ///
-/// Only use this projection for an authenticated, generation-fenced
-/// `refused_before_start` report; ordinary runtime errors are not startup claims.
+/// Only use this projection for an authenticated, generation-fenced startup
+/// report. An unconfirmed launch does not prove that workload execution stopped.
 #[must_use]
 pub fn backend_startup_diagnostic(code: &str) -> Option<String> {
     let (stage, kind) = code.strip_prefix("openshell.backend.")?.split_once('.')?;
+    if stage == "start-agent" {
+        return (kind == "unconfirmed").then(|| LAUNCH_UNCONFIRMED_DIAGNOSTIC.to_string());
+    }
     let stage = match stage {
         "build" => "client construction",
         "attach" => "attachment",

crates/openshell-driver-docker/src/lib.rs

@@ -23,6 +23,10 @@ use bollard::query_parameters::{
 };
 use bytes::Bytes;
 use futures::{Stream, StreamExt};
+use openshell_core::backend_startup::{
+    LAUNCH_UNCONFIRMED_CONDITION, LAUNCH_UNCONFIRMED_DIAGNOSTIC, RuntimeStartupFailure,
+    SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED,
+};
 use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS;
 use openshell_core::driver_mounts;
 use openshell_core::driver_utils::{
@@ -5653,7 +5657,7 @@ async fn wait_for_docker_supervisor_ready(
         .await
     {
         Ok(()) => Ok(()),
-        Err(status) => Err(recorded_workspace_validation_failure(failures, sandbox_id)
+        Err(status) => Err(recorded_supervisor_start_failure(failures, sandbox_id)
             .await
             .unwrap_or(status)),
     }
@@ -5675,7 +5679,14 @@ async fn poll_docker_supervisor_ready(
             .map_err(|error| {
                 Status::internal(format!("inspect Docker sandbox container: {error}"))
             })?;
-        if sandbox.state.unwrap_or_default().running == Some(false) {
+        let workload_state = sandbox.state.unwrap_or_default();
+        if workload_state.running == Some(false) {
+            if let Some(failure) = workload_state
+                .exit_code
+                .and_then(RuntimeStartupFailure::from_exit_code)
+            {
+                return Err(Status::failed_precondition(failure.diagnostic()));
+            }
             return Err(Status::unavailable(
                 "Docker sandbox exited before supervisor became ready",
             ));
@@ -5697,6 +5708,11 @@ async fn poll_docker_supervisor_ready(
                 warn!(sandbox_id, supervisor_id, supervisor_logs = %log_tail, "Docker sandbox rejected the image working directory");
                 return Err(workspace_validation_status());
             }
+            _ if state.running == Some(false)
+                && state.exit_code == Some(i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED)) =>
+            {
+                return Err(Status::unavailable(LAUNCH_UNCONFIRMED_DIAGNOSTIC));
+            }
             _ if state.running == Some(false) => {
                 let log_tail = docker_container_log_tail(docker, supervisor_id).await;
                 warn!(sandbox_id, supervisor_id, supervisor_logs = %log_tail, "Docker supervisor exited before becoming ready");
@@ -5716,6 +5732,20 @@ fn docker_supervisor_wait_failure(
     result: Option<Result<ContainerWaitResponse, BollardError>>,
 ) -> (&'static str, String) {
     match result {
+        Some(Ok(status)) if status.status_code == i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED) => {
+            (
+                LAUNCH_UNCONFIRMED_CONDITION,
+                LAUNCH_UNCONFIRMED_DIAGNOSTIC.to_string(),
+            )
+        }
+        Some(Err(BollardError::DockerContainerWaitError { code, .. }))
+            if code == i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED) =>
+        {
+            (
+                LAUNCH_UNCONFIRMED_CONDITION,
+                LAUNCH_UNCONFIRMED_DIAGNOSTIC.to_string(),
+            )
+        }
         Some(Ok(status)) => (
             supervisor_exit_reason(status.status_code),
             format!(
@@ -5738,15 +5768,20 @@ fn docker_supervisor_wait_failure(
     }
 }
 
-async fn recorded_workspace_validation_failure(
+async fn recorded_supervisor_start_failure(
     failures: &Arc<Mutex<HashMap<String, DockerRuntimeFailure>>>,
     sandbox_id: &str,
 ) -> Option<Status> {
     let failures = failures.lock().await;
     failures
         .get(sandbox_id)
-        .filter(|failure| failure.reason == CONDITION_WORKSPACE_VALIDATION_FAILED)
-        .map(|_| workspace_validation_status())
+        .and_then(|failure| match failure.reason {
+            CONDITION_WORKSPACE_VALIDATION_FAILED => Some(workspace_validation_status()),
+            LAUNCH_UNCONFIRMED_CONDITION => {
+                Some(Status::unavailable(LAUNCH_UNCONFIRMED_DIAGNOSTIC))
+            }
+            _ => None,
+        })
 }
 
 /// Startup failure for a supervisor that exited because the sandbox rejected
@@ -5763,15 +5798,31 @@ fn supervisor_start_failure_reason(status: &Status, default: &'static str) -> &'
         && status.message() == WORKSPACE_VALIDATION_FAILED_MESSAGE
     {
         CONDITION_WORKSPACE_VALIDATION_FAILED
+    } else if status.code() == tonic::Code::Unavailable
+        && status.message() == LAUNCH_UNCONFIRMED_DIAGNOSTIC
+    {
+        LAUNCH_UNCONFIRMED_CONDITION
     } else {
-        default
+        [
+            RuntimeStartupFailure::LandlockUnavailable,
+            RuntimeStartupFailure::LandlockEnforcement,
+            RuntimeStartupFailure::Qualification,
+        ]
+        .into_iter()
+        .find(|failure| {
+            status.code() == tonic::Code::FailedPrecondition
+                && status.message() == failure.diagnostic()
+        })
+        .map_or(default, RuntimeStartupFailure::condition_reason)
     }
 }
 
 /// Condition reason for a supervisor that exited with `status_code`.
 fn supervisor_exit_reason(status_code: i64) -> &'static str {
     if status_code == i64::from(SUPERVISOR_EXIT_WORKSPACE_VALIDATION_FAILED) {
         CONDITION_WORKSPACE_VALIDATION_FAILED
+    } else if status_code == i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED) {
+        LAUNCH_UNCONFIRMED_CONDITION
     } else {
         "ControlSupervisorExited"
     }
@@ -6576,7 +6627,7 @@ fn apply_docker_exit_classification(sandbox: &mut DriverSandbox, state: &Contain
     if state.oom_killed == Some(true) {
         return;
     }
-    let Some(code) = state.exit_code.filter(|&code| matches!(code, 137 | 143)) else {
+    let Some(code) = state.exit_code else {
         return;
     };
     let Some(condition) = sandbox
@@ -6589,8 +6640,13 @@ fn apply_docker_exit_classification(sandbox: &mut DriverSandbox, state: &Contain
     if condition.reason != CONDITION_EXITED {
         return;
     }
-    condition.reason = CONDITION_RUNTIME_RESTART.to_string();
-    condition.message = format!("Container terminated by signal (exit code {code})");
+    if let Some(failure) = RuntimeStartupFailure::from_exit_code(code) {
+        condition.reason = failure.condition_reason().to_string();
+        condition.message = failure.diagnostic().to_string();
+    } else if matches!(code, 137 | 143) {
+        condition.reason = CONDITION_RUNTIME_RESTART.to_string();
+        condition.message = format!("Container terminated by signal (exit code {code})");
+    }
 }
 
 fn container_ready_condition(

crates/openshell-driver-docker/src/tests.rs

@@ -4378,6 +4378,29 @@ fn docker_oom_kill_stays_terminal_despite_137() {
     assert_eq!(ready_reason(&sandbox), CONDITION_EXITED);
 }
 
+#[test]
+fn runtime_qualification_exit_reports_fixed_failure() {
+    for failure in [
+        RuntimeStartupFailure::LandlockUnavailable,
+        RuntimeStartupFailure::LandlockEnforcement,
+        RuntimeStartupFailure::Qualification,
+    ] {
+        let mut sandbox = exited_sandbox_with_ready_reason(CONDITION_EXITED);
+        let state = ContainerState {
+            status: Some(ContainerStateStatusEnum::EXITED),
+            exit_code: Some(i64::from(failure.exit_code())),
+            ..Default::default()
+        };
+        apply_docker_exit_classification(&mut sandbox, &state);
+        assert_eq!(ready_reason(&sandbox), failure.condition_reason());
+        let status = Status::failed_precondition(failure.diagnostic());
+        assert_eq!(
+            supervisor_start_failure_reason(&status, "fallback"),
+            failure.condition_reason()
+        );
+    }
+}
+
 #[test]
 fn supervisor_workspace_validation_exit_is_reported_explicitly() {
     let status = workspace_validation_status();
@@ -4398,6 +4421,35 @@ fn supervisor_workspace_validation_exit_is_reported_explicitly() {
         CONDITION_WORKSPACE_VALIDATION_FAILED
     );
     assert_eq!(supervisor_exit_reason(1), "ControlSupervisorExited");
+    assert_eq!(
+        supervisor_exit_reason(i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED)),
+        LAUNCH_UNCONFIRMED_CONDITION
+    );
+    assert_eq!(
+        supervisor_start_failure_reason(
+            &Status::unavailable(LAUNCH_UNCONFIRMED_DIAGNOSTIC),
+            "fallback"
+        ),
+        LAUNCH_UNCONFIRMED_CONDITION
+    );
+    for wait in [
+        Some(Ok(ContainerWaitResponse {
+            status_code: i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED),
+            ..Default::default()
+        })),
+        Some(Err(BollardError::DockerContainerWaitError {
+            code: i64::from(SUPERVISOR_EXIT_LAUNCH_UNCONFIRMED),
+            error: "PRIVATE STDERR".into(),
+        })),
+    ] {
+        assert_eq!(
+            docker_supervisor_wait_failure(wait),
+            (
+                LAUNCH_UNCONFIRMED_CONDITION,
+                LAUNCH_UNCONFIRMED_DIAGNOSTIC.to_string()
+            )
+        );
+    }
 
     let (reason, message) =
         docker_supervisor_wait_failure(Some(Err(BollardError::DockerContainerWaitError {