feat(compute)!: configure provisioning and bootstrap timeouts

crates/openshell-core/src/config.rs

@@ -328,6 +328,11 @@ pub struct Config {
     /// sandbox attempts, in seconds. Must be between 1 and 86400, inclusive.
     pub image_preparation_timeout_seconds: u32,
 
+    /// Provisioning repair window, in seconds. The first authenticated
+    /// supervisor configuration report starts admission repair, which stays
+    /// armed until Ready. Must be between 1 and 86400, inclusive.
+    pub provisioning_repair_window_seconds: i64,
+
     /// Maximum gRPC requests allowed per rate-limit window.
     ///
     /// When paired with [`Self::grpc_rate_limit_window_secs`], positive values
@@ -1048,6 +1053,7 @@ impl Config {
             default_credential_driver: None,
             ssh_session_ttl_secs: default_ssh_session_ttl_secs(),
             image_preparation_timeout_seconds: 1800,
+            provisioning_repair_window_seconds: 300,
             grpc_rate_limit_requests: None,
             grpc_rate_limit_window_secs: None,
             service_routing: ServiceRoutingConfig::default(),

crates/openshell-driver-kubernetes/src/config.rs

@@ -346,6 +346,17 @@ pub struct KubernetesComputeConfig {
     pub supervisor_image_pull_policy: Option<KubernetesImagePullPolicy>,
     /// Cross-pod sandbox/supervisor settings.
     pub sandbox_runtime: KubernetesSandboxRuntimeConfig,
+    /// Bound on how long a crash-interrupted, fail-closed sandbox runtime
+    /// bootstrap may remain stranded before the driver reclaims it, in seconds. Slow
+    /// VM-backed runtimes (CoCo/peer-pods) need longer than the default.
+    pub runtime_bootstrap_grace_seconds: u64,
+    /// Supervisor attach/policy-discover request timeout, in seconds.
+    /// When set, the driver copies the value into the supervisor pod
+    /// environment (`OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS`) so slow VM-backed
+    /// runtimes have time to boot the remote boundary before the supervisor
+    /// gives up.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub sandbox_attach_request_timeout_seconds: Option<u64>,
     /// Corporate HTTP forward proxy used by the network supervisor for
     /// policy-approved TLS CONNECT egress.
     pub https_proxy: Option<String>,
@@ -509,6 +520,8 @@ impl Default for KubernetesComputeConfig {
             supervisor_image: config::default_supervisor_image(),
             supervisor_image_pull_policy: None,
             sandbox_runtime: KubernetesSandboxRuntimeConfig::default(),
+            runtime_bootstrap_grace_seconds: 300,
+            sandbox_attach_request_timeout_seconds: None,
             https_proxy: None,
             no_proxy: None,
             proxy_auth_secret_name: None,
@@ -539,6 +552,7 @@ impl Default for KubernetesComputeConfig {
 impl KubernetesComputeConfig {
     /// Validate startup configuration without connecting to Kubernetes.
     pub fn validate_configuration(&self) -> Result<(), String> {
+        self.validate_bootstrap_windows()?;
         self.validate_supervisor_topology()?;
         self.validate_workspace_mode()?;
         self.validate_provider_spiffe_workload_api_socket_path()?;
@@ -549,6 +563,24 @@ impl KubernetesComputeConfig {
         self.validate_allowed_pod_labels()
     }
 
+    fn validate_bootstrap_windows(&self) -> Result<(), String> {
+        for (name, value) in [
+            (
+                "runtime_bootstrap_grace_seconds",
+                Some(self.runtime_bootstrap_grace_seconds),
+            ),
+            (
+                "sandbox_attach_request_timeout_seconds",
+                self.sandbox_attach_request_timeout_seconds,
+            ),
+        ] {
+            if value.is_some_and(|value| !(1..=86_400).contains(&value)) {
+                return Err(format!("{name} must be between 1 and 86400 seconds"));
+            }
+        }
+        Ok(())
+    }
+
     /// The shared supervisor serves only sandbox instances it can isolate: it
     /// holds no per-sandbox workload identity, upstream proxy credential, or
     /// proxy CA.
@@ -1244,6 +1276,43 @@ mod tests {
         }
     }
 
+    #[test]
+    fn bootstrap_grace_defaults_and_parses_from_toml() {
+        assert_eq!(
+            KubernetesComputeConfig::default().runtime_bootstrap_grace_seconds,
+            300
+        );
+        assert!(
+            KubernetesComputeConfig::default()
+                .sandbox_attach_request_timeout_seconds
+                .is_none()
+        );
+        let config: KubernetesComputeConfig = toml::from_str(
+            "runtime_bootstrap_grace_seconds = 900\nsandbox_attach_request_timeout_seconds = 900\n",
+        )
+        .unwrap();
+        assert_eq!(config.runtime_bootstrap_grace_seconds, 900);
+        assert_eq!(config.sandbox_attach_request_timeout_seconds, Some(900));
+        assert!(config.validate_configuration().is_ok());
+        let partial: KubernetesComputeConfig = toml::from_str("namespace = 'custom'").unwrap();
+        assert_eq!(partial.runtime_bootstrap_grace_seconds, 300);
+        for field in [
+            "runtime_bootstrap_grace_seconds",
+            "sandbox_attach_request_timeout_seconds",
+        ] {
+            for value in [0, 86_401] {
+                let config: KubernetesComputeConfig =
+                    toml::from_str(&format!("{field} = {value}")).unwrap();
+                assert!(config.validate_configuration().unwrap_err().contains(field));
+            }
+            for value in [1, 86_400] {
+                let config: KubernetesComputeConfig =
+                    toml::from_str(&format!("{field} = {value}")).unwrap();
+                assert!(config.validate_configuration().is_ok());
+            }
+        }
+    }
+
     #[test]
     fn published_kubernetes_example_is_valid_toml() {
         let docs = include_str!("../../../docs/how-it-works/gateways/configuration.mdx");

crates/openshell-driver-kubernetes/src/driver.rs

@@ -612,10 +612,6 @@ fn admission_error(error: tonic::Status) -> KubernetesDriverError {
     }
 }
 const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30);
-/// Unsupervised shared sandbox instances removed at once by one reconcile pass.
-/// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded.
-const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5);
-
 fn random_sandbox_runtime_token() -> Result<String, KubernetesDriverError> {
     use std::fmt::Write as _;
 
@@ -3583,6 +3579,7 @@ impl KubernetesComputeDriver {
                         .map(|generation| (generation.name.as_str(), generation.digest.as_str())),
                     owner,
                     &self.supervisor_otlp_environment(),
+                    self.config.sandbox_attach_request_timeout_seconds,
                 )
                 .map_err(KubernetesDriverError::Message)?,
                 &self.config.sandbox_runtime.app_listener,
@@ -4102,6 +4099,7 @@ impl KubernetesComputeDriver {
             sandbox_api.api.clone(),
             cr_name.to_string(),
             Some(cr_uid.to_string()),
+            Duration::from_secs(self.config.runtime_bootstrap_grace_seconds),
         );
         Ok(())
     }
@@ -5543,7 +5541,7 @@ impl KubernetesComputeDriver {
         if !sandbox_runtime_bootstrap_is_stale(
             object,
             SystemTime::now(),
-            SANDBOX_RUNTIME_BOOTSTRAP_GRACE,
+            Duration::from_secs(self.config.runtime_bootstrap_grace_seconds),
         ) {
             return;
         }
@@ -5986,9 +5984,10 @@ fn spawn_sandbox_runtime_bootstrap_completion(
     sandboxes: Api<DynamicObject>,
     sandbox_name: String,
     expected_sandbox_uid: Option<String>,
+    bootstrap_grace: Duration,
 ) {
     tokio::spawn(async move {
-        let deadline = tokio::time::Instant::now() + SANDBOX_RUNTIME_BOOTSTRAP_GRACE;
+        let deadline = tokio::time::Instant::now() + bootstrap_grace;
         match tokio::time::timeout_at(deadline, control.wait_available()).await {
             Ok(true) => {}
             Ok(false) | Err(_) => return,
@@ -10513,6 +10512,7 @@ mod tests {
     #[test]
     fn sandbox_runtime_bootstrap_marker_and_age_gate_rollback() {
         let started = Duration::from_hours(490_896);
+        let bootstrap_grace = Duration::from_mins(5);
         let mut object: DynamicObject = serde_json::from_value(serde_json::json!({
             "apiVersion": "agents.x-k8s.io/v1beta1",
             "kind": "Sandbox",
@@ -10535,13 +10535,13 @@ mod tests {
         let started_at = SystemTime::UNIX_EPOCH + started;
         assert!(!sandbox_runtime_bootstrap_is_stale(
             &object,
-            started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE - Duration::from_secs(1),
-            SANDBOX_RUNTIME_BOOTSTRAP_GRACE
+            started_at + bootstrap_grace - Duration::from_secs(1),
+            bootstrap_grace
         ));
         assert!(sandbox_runtime_bootstrap_is_stale(
             &object,
-            started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE,
-            SANDBOX_RUNTIME_BOOTSTRAP_GRACE
+            started_at + bootstrap_grace,
+            bootstrap_grace
         ));
         object
             .metadata
@@ -10552,7 +10552,7 @@ mod tests {
         assert!(sandbox_runtime_bootstrap_is_stale(
             &object,
             started_at,
-            SANDBOX_RUNTIME_BOOTSTRAP_GRACE
+            bootstrap_grace
         ));
         object
             .metadata

crates/openshell-driver-kubernetes/src/main.rs

@@ -29,6 +29,14 @@ struct Args {
         default_value = "{}"
     )]
     admission_config_json: openshell_core::resource_admission::DriverAdmissionConfig,
+    /// Runtime bootstrap budget in seconds.
+    #[arg(long, env = "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_SECONDS", default_value = "300",
+        value_parser = clap::value_parser!(u64).range(1..=86_400))]
+    runtime_bootstrap_grace_seconds: u64,
+    /// Initial attachment budget passed to dedicated supervisor Pods.
+    #[arg(long, env = "OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS",
+        value_parser = clap::value_parser!(u64).range(1..=86_400))]
+    sandbox_attach_request_timeout_seconds: Option<u64>,
     /// Public compute-driver Unix socket used by an external gateway.
     #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")]
     bind_socket: Option<PathBuf>,
@@ -286,6 +294,8 @@ async fn main() -> Result<()> {
                     openshell_driver_kubernetes::config::KubernetesAppListenerConfig::default(),
                 boundary_port: args.sandbox_runtime_boundary_port,
             },
+            runtime_bootstrap_grace_seconds: args.runtime_bootstrap_grace_seconds,
+            sandbox_attach_request_timeout_seconds: args.sandbox_attach_request_timeout_seconds,
             https_proxy: args.https_proxy,
             no_proxy: args.no_proxy,
             proxy_auth_secret_name: args.proxy_auth_secret_name,
@@ -365,6 +375,33 @@ async fn main() -> Result<()> {
 
 #[cfg(test)]
 mod tests {
+    #[test]
+    fn bootstrap_timeout_arguments_reject_invalid_values() {
+        let defaults = Args::try_parse_from(["openshell-driver-kubernetes"]).unwrap();
+        assert_eq!(defaults.runtime_bootstrap_grace_seconds, 300);
+        assert!(defaults.sandbox_attach_request_timeout_seconds.is_none());
+        for flag in [
+            "--runtime-bootstrap-grace-seconds",
+            "--sandbox-attach-request-timeout-seconds",
+        ] {
+            for value in ["0", "86401", "invalid"] {
+                assert!(
+                    Args::try_parse_from(["openshell-driver-kubernetes", flag, value]).is_err()
+                );
+            }
+            for value in ["1", "900", "86400"] {
+                let args =
+                    Args::try_parse_from(["openshell-driver-kubernetes", flag, value]).unwrap();
+                let seconds = value.parse::<u64>().unwrap();
+                if flag == "--runtime-bootstrap-grace-seconds" {
+                    assert_eq!(args.runtime_bootstrap_grace_seconds, seconds);
+                } else {
+                    assert_eq!(args.sandbox_attach_request_timeout_seconds, Some(seconds));
+                }
+            }
+        }
+    }
+
     use super::*;
 
     #[test]

crates/openshell-driver-kubernetes/src/sandbox_runtime.rs

@@ -353,6 +353,7 @@ pub fn supervisor_pod(
     network_additional_ca: Option<(&str, &str)>,
     owner: OwnerReference,
     tracing_environment: &[(&str, String)],
+    attach_request_timeout_seconds: Option<u64>,
 ) -> Result<Pod, String> {
     let labels = control_labels(sandbox_id, gateway_id);
     let mut environment = vec![
@@ -394,6 +395,12 @@ pub fn supervisor_pod(
             .iter()
             .map(|(name, value)| env_var(name, value)),
     );
+    if let Some(timeout_seconds) = attach_request_timeout_seconds {
+        environment.push(env_var(
+            "OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS",
+            &timeout_seconds.to_string(),
+        ));
+    }
     if let Some(server_name) = gateway_tls_server_name(grpc_endpoint) {
         environment.push(env_var(
             openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME,
@@ -1045,6 +1052,7 @@ mod tests {
             None,
             owner(),
             &[],
+            None,
         )
         .expect("render supervisor Pod")
     }
@@ -1111,6 +1119,7 @@ mod tests {
                 ),
                 ("TRACEPARENT", TRACEPARENT.to_string()),
             ],
+            None,
         )
         .expect("render supervisor Pod");
         let environment = pod.spec.expect("Pod spec").containers[0]
@@ -1127,6 +1136,56 @@ mod tests {
         assert_eq!(environment["TRACEPARENT"], TRACEPARENT);
     }
 
+    #[test]
+    fn supervisor_pod_sets_attach_request_timeout_only_when_configured() {
+        let env = |timeout: Option<u64>| {
+            supervisor_pod(
+                "sandbox",
+                &SandboxRuntimeNames::new("pair"),
+                "pair",
+                "demo",
+                "gateway",
+                "supervisor:latest",
+                None,
+                "sandbox-sa",
+                1000,
+                1000,
+                &[],
+                "https://gateway:8080",
+                SupervisorClientTls::Disabled,
+                "{}",
+                "info",
+                600,
+                None,
+                None,
+                None,
+                false,
+                false,
+                false,
+                None,
+                None,
+                owner(),
+                &[],
+                timeout,
+            )
+            .unwrap()
+            .spec
+            .expect("Pod spec")
+            .containers[0]
+                .env
+                .clone()
+                .unwrap_or_default()
+                .into_iter()
+                .map(|variable| (variable.name, variable.value.unwrap_or_default()))
+                .collect::<BTreeMap<_, _>>()
+        };
+        assert!(!env(None).contains_key("OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS"));
+        assert_eq!(
+            env(Some(900))["OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS"],
+            "900"
+        );
+    }
+
     #[test]
     fn supervisor_pod_reads_client_tls_from_the_bootstrap_secret() {
         let (env, volumes) = tls_env_and_volumes(&supervisor_pod_with_client_tls(
@@ -1232,6 +1291,7 @@ mod tests {
             None,
             owner(),
             &[],
+            None,
         )
         .expect("render supervisor Pod");
         let pod_spec = pod.spec.as_ref().expect("Pod spec");
@@ -1401,6 +1461,7 @@ mod tests {
                 Some(("openshell-additional-ca-gateway-generation", digest)),
                 owner(),
                 &[],
+                None,
             )
             .expect("render supervisor Pod with destination trust");
             let pod = crate::app_listener::configure_supervisor_pod(
@@ -1593,6 +1654,7 @@ mod tests {
             None,
             owner(),
             &[],
+            None,
         )
         .expect("render supervisor Pod")
     }