From 739b0f98b2609de8763e192421520cd22f3e275d Mon Sep 17 00:00:00 2001 From: Shiju Date: Wed, 4 Nov 2026 15:59:37 +0530 Subject: [PATCH 1/3] feat(compute)!: configure provisioning and bootstrap timeouts Continue Mike Hepburn's configuration contribution from NVIDIA/OpenShell#4575 (de949b7738b9800ce88ef23ef78763ad165cae5a), preserving its attribution and original DCO provenance. Adapt it to current session admission, registered and shared backends, bound configuration input, and use the configured budget for the initial routed connection. Add focused timeout and configuration regressions. These adaptations are Shiju's work and are not represented as endorsed by the original contributor. BREAKING CHANGE: External Rust configuration struct literals must supply the new timeout fields. PreparedBackend discovery implementors and callers of public supervisor/runtime policy-discovery methods must accept or pass the timeout argument. Target the next qualified minor release, not a patch release or backport. Existing TOML defaults and normal reconnect limits are retained. Co-authored-by: Mike Hepburn Signed-off-by: Mike Hepburn Signed-off-by: Shiju --- crates/openshell-core/src/config.rs | 6 + .../openshell-driver-kubernetes/src/config.rs | 71 ++++++ .../openshell-driver-kubernetes/src/driver.rs | 22 +- .../openshell-driver-kubernetes/src/main.rs | 27 +++ .../src/sandbox_runtime.rs | 62 ++++++ crates/openshell-gateway/src/lib.rs | 26 +++ crates/openshell-sandbox-backend/Cargo.toml | 1 + .../openshell-sandbox-backend/src/runtime.rs | 150 ++++++++++++- .../src/runtime/tests/delegated.rs | 6 +- crates/openshell-server/src/cli.rs | 136 ++++++++++++ crates/openshell-server/src/compute/mod.rs | 146 ++++++++---- .../src/compute/provisioning_deadline.rs | 208 ++++++++++++------ .../src/compute/tests/error_recovery.rs | 6 +- .../src/compute/tests/suspend_resume.rs | 9 +- crates/openshell-server/src/config_file.rs | 3 + crates/openshell-server/src/grpc/policy.rs | 22 +- crates/openshell-server/src/grpc/sandbox.rs | 2 + crates/openshell-server/src/lib.rs | 3 + .../src/persistence/sandbox_image.rs | 3 +- .../openshell-supervisor/src/backend_setup.rs | 8 +- .../src/backend_setup/registration.rs | 21 +- .../src/backend_setup/tests.rs | 4 + .../src/legacy_startup_regression_tests.rs | 2 + crates/openshell-supervisor/src/lib.rs | 7 +- crates/openshell-supervisor/src/main.rs | 37 ++++ crates/openshell-supervisor/src/shared.rs | 3 + .../openshell-supervisor/src/shared_host.rs | 11 + docs/how-it-works/gateways/configuration.mdx | 28 +++ skills/debug-openshell-cluster/SKILL.md | 10 +- 29 files changed, 890 insertions(+), 150 deletions(-) diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 20f6a334b1..bcf382ce1e 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -328,6 +328,11 @@ pub struct Config { /// sandbox attempts, in seconds. Must be between 1 and 86400, inclusive. pub image_preparation_timeout_seconds: u32, + /// Provisioning repair window, in milliseconds. The first authenticated + /// supervisor configuration report starts admission repair, which stays + /// armed until Ready. Must be between 1000 and 86400000, inclusive. + pub provisioning_repair_window_ms: i64, + /// Maximum gRPC requests allowed per rate-limit window. /// /// When paired with [`Self::grpc_rate_limit_window_secs`], positive values @@ -1048,6 +1053,7 @@ impl Config { default_credential_driver: None, ssh_session_ttl_secs: default_ssh_session_ttl_secs(), image_preparation_timeout_seconds: 1800, + provisioning_repair_window_ms: 300_000, grpc_rate_limit_requests: None, grpc_rate_limit_window_secs: None, service_routing: ServiceRoutingConfig::default(), diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 28d3161f42..1fa91e34cf 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -346,6 +346,17 @@ pub struct KubernetesComputeConfig { pub supervisor_image_pull_policy: Option, /// Cross-pod sandbox/supervisor settings. pub sandbox_runtime: KubernetesSandboxRuntimeConfig, + /// Bound on how long a crash-interrupted, fail-closed sandbox runtime + /// bootstrap may remain stranded before the driver reclaims it. Slow + /// VM-backed runtimes (CoCo/peer-pods) need longer than the default. + pub runtime_bootstrap_grace_ms: u64, + /// Supervisor attach/policy-discover request timeout, in milliseconds. + /// When set, the driver copies the value into the supervisor pod + /// environment (`OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS`) so slow VM-backed + /// runtimes have time to boot the remote boundary before the supervisor + /// gives up. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sandbox_attach_request_timeout_ms: Option, /// Corporate HTTP forward proxy used by the network supervisor for /// policy-approved TLS CONNECT egress. pub https_proxy: Option, @@ -509,6 +520,8 @@ impl Default for KubernetesComputeConfig { supervisor_image: config::default_supervisor_image(), supervisor_image_pull_policy: None, sandbox_runtime: KubernetesSandboxRuntimeConfig::default(), + runtime_bootstrap_grace_ms: 300_000, + sandbox_attach_request_timeout_ms: None, https_proxy: None, no_proxy: None, proxy_auth_secret_name: None, @@ -539,6 +552,7 @@ impl Default for KubernetesComputeConfig { impl KubernetesComputeConfig { /// Validate startup configuration without connecting to Kubernetes. pub fn validate_configuration(&self) -> Result<(), String> { + self.validate_bootstrap_windows()?; self.validate_supervisor_topology()?; self.validate_workspace_mode()?; self.validate_provider_spiffe_workload_api_socket_path()?; @@ -549,6 +563,26 @@ impl KubernetesComputeConfig { self.validate_allowed_pod_labels() } + fn validate_bootstrap_windows(&self) -> Result<(), String> { + for (name, value) in [ + ( + "runtime_bootstrap_grace_ms", + Some(self.runtime_bootstrap_grace_ms), + ), + ( + "sandbox_attach_request_timeout_ms", + self.sandbox_attach_request_timeout_ms, + ), + ] { + if value.is_some_and(|value| !(1000..=86_400_000).contains(&value)) { + return Err(format!( + "{name} must be between 1000 and 86400000 milliseconds" + )); + } + } + Ok(()) + } + /// The shared supervisor serves only sandbox instances it can isolate: it /// holds no per-sandbox workload identity, upstream proxy credential, or /// proxy CA. @@ -1244,6 +1278,43 @@ mod tests { } } + #[test] + fn bootstrap_grace_defaults_and_parses_from_toml() { + assert_eq!( + KubernetesComputeConfig::default().runtime_bootstrap_grace_ms, + 300_000 + ); + assert!( + KubernetesComputeConfig::default() + .sandbox_attach_request_timeout_ms + .is_none() + ); + let config: KubernetesComputeConfig = toml::from_str( + "runtime_bootstrap_grace_ms = 900000\nsandbox_attach_request_timeout_ms = 900000\n", + ) + .unwrap(); + assert_eq!(config.runtime_bootstrap_grace_ms, 900_000); + assert_eq!(config.sandbox_attach_request_timeout_ms, Some(900_000)); + assert!(config.validate_configuration().is_ok()); + let partial: KubernetesComputeConfig = toml::from_str("namespace = 'custom'").unwrap(); + assert_eq!(partial.runtime_bootstrap_grace_ms, 300_000); + for field in [ + "runtime_bootstrap_grace_ms", + "sandbox_attach_request_timeout_ms", + ] { + for value in [0, 999, 86_400_001] { + let config: KubernetesComputeConfig = + toml::from_str(&format!("{field} = {value}")).unwrap(); + assert!(config.validate_configuration().unwrap_err().contains(field)); + } + for value in [1000, 86_400_000] { + let config: KubernetesComputeConfig = + toml::from_str(&format!("{field} = {value}")).unwrap(); + assert!(config.validate_configuration().is_ok()); + } + } + } + #[test] fn published_kubernetes_example_is_valid_toml() { let docs = include_str!("../../../docs/how-it-works/gateways/configuration.mdx"); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index c08a567eed..7afe3e4579 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -612,10 +612,6 @@ fn admission_error(error: tonic::Status) -> KubernetesDriverError { } } const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); -/// Unsupervised shared sandbox instances removed at once by one reconcile pass. -/// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded. -const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5); - fn random_sandbox_runtime_token() -> Result { use std::fmt::Write as _; @@ -3583,6 +3579,7 @@ impl KubernetesComputeDriver { .map(|generation| (generation.name.as_str(), generation.digest.as_str())), owner, &self.supervisor_otlp_environment(), + self.config.sandbox_attach_request_timeout_ms, ) .map_err(KubernetesDriverError::Message)?, &self.config.sandbox_runtime.app_listener, @@ -4102,6 +4099,7 @@ impl KubernetesComputeDriver { sandbox_api.api.clone(), cr_name.to_string(), Some(cr_uid.to_string()), + Duration::from_millis(self.config.runtime_bootstrap_grace_ms), ); Ok(()) } @@ -5543,7 +5541,7 @@ impl KubernetesComputeDriver { if !sandbox_runtime_bootstrap_is_stale( object, SystemTime::now(), - SANDBOX_RUNTIME_BOOTSTRAP_GRACE, + Duration::from_millis(self.config.runtime_bootstrap_grace_ms), ) { return; } @@ -5986,9 +5984,10 @@ fn spawn_sandbox_runtime_bootstrap_completion( sandboxes: Api, sandbox_name: String, expected_sandbox_uid: Option, + bootstrap_grace: Duration, ) { tokio::spawn(async move { - let deadline = tokio::time::Instant::now() + SANDBOX_RUNTIME_BOOTSTRAP_GRACE; + let deadline = tokio::time::Instant::now() + bootstrap_grace; match tokio::time::timeout_at(deadline, control.wait_available()).await { Ok(true) => {} Ok(false) | Err(_) => return, @@ -10513,6 +10512,7 @@ mod tests { #[test] fn sandbox_runtime_bootstrap_marker_and_age_gate_rollback() { let started = Duration::from_hours(490_896); + let bootstrap_grace = Duration::from_mins(5); let mut object: DynamicObject = serde_json::from_value(serde_json::json!({ "apiVersion": "agents.x-k8s.io/v1beta1", "kind": "Sandbox", @@ -10535,13 +10535,13 @@ mod tests { let started_at = SystemTime::UNIX_EPOCH + started; assert!(!sandbox_runtime_bootstrap_is_stale( &object, - started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE - Duration::from_secs(1), - SANDBOX_RUNTIME_BOOTSTRAP_GRACE + started_at + bootstrap_grace - Duration::from_secs(1), + bootstrap_grace )); assert!(sandbox_runtime_bootstrap_is_stale( &object, - started_at + SANDBOX_RUNTIME_BOOTSTRAP_GRACE, - SANDBOX_RUNTIME_BOOTSTRAP_GRACE + started_at + bootstrap_grace, + bootstrap_grace )); object .metadata @@ -10552,7 +10552,7 @@ mod tests { assert!(sandbox_runtime_bootstrap_is_stale( &object, started_at, - SANDBOX_RUNTIME_BOOTSTRAP_GRACE + bootstrap_grace )); object .metadata diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 69ffc69ccb..af1669ef03 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -29,6 +29,14 @@ struct Args { default_value = "{}" )] admission_config_json: openshell_core::resource_admission::DriverAdmissionConfig, + /// Runtime bootstrap budget in milliseconds. + #[arg(long, env = "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS", default_value = "300000", + value_parser = clap::value_parser!(u64).range(1000..=86_400_000))] + runtime_bootstrap_grace_ms: u64, + /// Initial attachment budget passed to dedicated supervisor Pods. + #[arg(long, env = "OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS", + value_parser = clap::value_parser!(u64).range(1000..=86_400_000))] + sandbox_attach_request_timeout_ms: Option, /// Public compute-driver Unix socket used by an external gateway. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] bind_socket: Option, @@ -286,6 +294,8 @@ async fn main() -> Result<()> { openshell_driver_kubernetes::config::KubernetesAppListenerConfig::default(), boundary_port: args.sandbox_runtime_boundary_port, }, + runtime_bootstrap_grace_ms: args.runtime_bootstrap_grace_ms, + sandbox_attach_request_timeout_ms: args.sandbox_attach_request_timeout_ms, https_proxy: args.https_proxy, no_proxy: args.no_proxy, proxy_auth_secret_name: args.proxy_auth_secret_name, @@ -365,6 +375,23 @@ async fn main() -> Result<()> { #[cfg(test)] mod tests { + #[test] + fn bootstrap_timeout_arguments_reject_invalid_values() { + for flag in [ + "--runtime-bootstrap-grace-ms", + "--sandbox-attach-request-timeout-ms", + ] { + for value in ["0", "999", "86400001", "invalid"] { + assert!( + Args::try_parse_from(["openshell-driver-kubernetes", flag, value]).is_err() + ); + } + for value in ["1000", "900000", "86400000"] { + assert!(Args::try_parse_from(["openshell-driver-kubernetes", flag, value]).is_ok()); + } + } + } + use super::*; #[test] diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 39876b3d84..c596aa6391 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -353,6 +353,7 @@ pub fn supervisor_pod( network_additional_ca: Option<(&str, &str)>, owner: OwnerReference, tracing_environment: &[(&str, String)], + attach_request_timeout_ms: Option, ) -> Result { let labels = control_labels(sandbox_id, gateway_id); let mut environment = vec![ @@ -394,6 +395,12 @@ pub fn supervisor_pod( .iter() .map(|(name, value)| env_var(name, value)), ); + if let Some(timeout_ms) = attach_request_timeout_ms { + environment.push(env_var( + "OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS", + &timeout_ms.to_string(), + )); + } if let Some(server_name) = gateway_tls_server_name(grpc_endpoint) { environment.push(env_var( openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, @@ -1045,6 +1052,7 @@ mod tests { None, owner(), &[], + None, ) .expect("render supervisor Pod") } @@ -1111,6 +1119,7 @@ mod tests { ), ("TRACEPARENT", TRACEPARENT.to_string()), ], + None, ) .expect("render supervisor Pod"); let environment = pod.spec.expect("Pod spec").containers[0] @@ -1127,6 +1136,56 @@ mod tests { assert_eq!(environment["TRACEPARENT"], TRACEPARENT); } + #[test] + fn supervisor_pod_sets_attach_request_timeout_only_when_configured() { + let env = |timeout: Option| { + supervisor_pod( + "sandbox", + &SandboxRuntimeNames::new("pair"), + "pair", + "demo", + "gateway", + "supervisor:latest", + None, + "sandbox-sa", + 1000, + 1000, + &[], + "https://gateway:8080", + SupervisorClientTls::Disabled, + "{}", + "info", + 600, + None, + None, + None, + false, + false, + false, + None, + None, + owner(), + &[], + timeout, + ) + .unwrap() + .spec + .expect("Pod spec") + .containers[0] + .env + .clone() + .unwrap_or_default() + .into_iter() + .map(|variable| (variable.name, variable.value.unwrap_or_default())) + .collect::>() + }; + assert!(!env(None).contains_key("OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS")); + assert_eq!( + env(Some(900_000))["OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS"], + "900000" + ); + } + #[test] fn supervisor_pod_reads_client_tls_from_the_bootstrap_secret() { let (env, volumes) = tls_env_and_volumes(&supervisor_pod_with_client_tls( @@ -1232,6 +1291,7 @@ mod tests { None, owner(), &[], + None, ) .expect("render supervisor Pod"); let pod_spec = pod.spec.as_ref().expect("Pod spec"); @@ -1401,6 +1461,7 @@ mod tests { Some(("openshell-additional-ca-gateway-generation", digest)), owner(), &[], + None, ) .expect("render supervisor Pod with destination trust"); let pod = crate::app_listener::configure_supervisor_pod( @@ -1593,6 +1654,7 @@ mod tests { None, owner(), &[], + None, ) .expect("render supervisor Pod") } diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index a4b618cbc9..5327940bc7 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -342,6 +342,32 @@ fn kubernetes_config( config.supervisor_image, std::env::var(openshell_core::config::SUPERVISOR_IMAGE_ENV).ok(), ); + if let Some(raw) = std::env::var_os("OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS") { + let raw = raw.to_str().ok_or_else(|| { + openshell_core::Error::config( + "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS must be an integer number of milliseconds", + ) + })?; + let grace = raw.parse::().map_err(|_| { + openshell_core::Error::config( + "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS must be an integer number of milliseconds", + ) + })?; + config.runtime_bootstrap_grace_ms = grace; + } + if let Some(raw) = std::env::var_os("OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS") { + let raw = raw.to_str().ok_or_else(|| { + openshell_core::Error::config( + "OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS must be an integer number of milliseconds", + ) + })?; + let timeout = raw.parse::().map_err(|_| { + openshell_core::Error::config( + "OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS must be an integer number of milliseconds", + ) + })?; + config.sandbox_attach_request_timeout_ms = Some(timeout); + } if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { config.workspace_default_storage_size = size; } diff --git a/crates/openshell-sandbox-backend/Cargo.toml b/crates/openshell-sandbox-backend/Cargo.toml index 8e8ff7cb05..f95c7bd6a2 100644 --- a/crates/openshell-sandbox-backend/Cargo.toml +++ b/crates/openshell-sandbox-backend/Cargo.toml @@ -43,6 +43,7 @@ libc = "0.2" rustix = { workspace = true, features = ["fs", "process"] } [dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } jsonwebtoken = { workspace = true } [lints] diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index 9ea72d396c..e4c60c2d3e 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -52,10 +52,6 @@ const ATTACH_REQUEST_TIMEOUT: Duration = Duration::from_mins(5); /// callers retry whole calls above this; past boot, exhausting this window /// means the remote boundary (or its launcher) is gone rather than still starting. const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); -/// Bound on a routed sandbox instance's first connection. The instance is -/// registered before its workload Pod is scheduled and its image pulled; the -/// driver reaps a bootstrap that is still incomplete after the same interval. -const ROUTED_FIRST_CONNECT_TIMEOUT: Duration = Duration::from_mins(5); /// Bounds the liveness check that decides whether a stream failure lost the /// whole connection. const CONNECTION_PROBE_TIMEOUT: Duration = Duration::from_secs(5); @@ -159,6 +155,7 @@ impl BoundaryTransport { async fn connect( &self, descriptor: &SandboxRuntimeDescriptor, + initial_connect_timeout: Duration, ) -> Result { let connector = match &self.0 { TransportSource::Dial => None, @@ -176,7 +173,7 @@ impl BoundaryTransport { let window = if registration.has_connected() { CONNECT_RETRY_TIMEOUT } else { - ROUTED_FIRST_CONNECT_TIMEOUT + initial_connect_timeout }; return tokio::time::timeout(window, registration.accept(&descriptor.tls)) .await @@ -327,6 +324,7 @@ pub struct OpenShellRuntimeBackend { ca_file_paths: Arc>>, provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, sandbox_bearer: openshell_core::jwt::SessionBearerTokenSlot, + attach_request_timeout: Duration, delegated: Option, resume: Option, takeover: Option, @@ -355,9 +353,11 @@ impl OpenShellRuntimeBackend { descriptor: SandboxRuntimeDescriptor, bearer: openshell_core::jwt::SessionBearerTokenSlot, transport: BoundaryTransport, + attach_request_timeout: Duration, ) -> Result<(Option, bool), BackendError> { let mut client = BoundaryClient::new(descriptor, bearer); client.transport = transport; + client.attach_request_timeout = attach_request_timeout; client.discover_policy().await } @@ -371,6 +371,7 @@ impl OpenShellRuntimeBackend { ca_file_paths, provider_credentials, sandbox_bearer, + attach_request_timeout: ATTACH_REQUEST_TIMEOUT, delegated: None, resume: None, takeover: None, @@ -411,6 +412,7 @@ impl OpenShellRuntimeBackend { descriptor: SandboxRuntimeDescriptor, driver_descriptor: Vec, bearer: openshell_core::jwt::SessionBearerTokenSlot, + attach_request_timeout: Duration, ) -> Result<(Option, bool), BackendError> { Self::discover_registered_policy_with_resume( backend_name, @@ -418,6 +420,7 @@ impl OpenShellRuntimeBackend { driver_descriptor, bearer, None, + attach_request_timeout, ) .await } @@ -429,15 +432,24 @@ impl OpenShellRuntimeBackend { driver_descriptor: Vec, bearer: openshell_core::jwt::SessionBearerTokenSlot, resume: Option, + attach_request_timeout: Duration, ) -> Result<(Option, bool), BackendError> { crate::delegated::validate_backend_name(&backend_name)?; validate_runtime_coordinates(&descriptor)?; let mut client = BoundaryClient::with_backend(descriptor, bearer, backend_name, driver_descriptor); client.resume = resume; + client.attach_request_timeout = attach_request_timeout; client.discover_policy().await } + /// Set the initial attachment and image-policy discovery budget. + #[must_use] + pub fn with_attach_request_timeout(mut self, timeout: Duration) -> Self { + self.attach_request_timeout = timeout; + self + } + /// Use the transport selected for the descriptor this backend will attach. #[must_use] pub fn with_transport(mut self, transport: BoundaryTransport) -> Self { @@ -505,6 +517,7 @@ impl OpenShellRuntimeBackend { descriptor.payload.clone(), ); client.transport = self.transport.clone(); + client.attach_request_timeout = self.attach_request_timeout; client.resume.clone_from(&self.resume); if let Some(takeover) = &self.takeover { client.supervisor_instance_id = takeover.supervisor_instance_id; @@ -1584,6 +1597,7 @@ struct BoundaryClient { backend_supports_suspend: AtomicBool, supervisor_instance_id: crate::boundary_protocol::SupervisorInstanceId, sandbox_bearer: openshell_core::jwt::SessionBearerTokenSlot, + attach_request_timeout: Duration, grpc_channel: tokio::sync::Mutex>, mediation: tokio::sync::Mutex>>, mediation_open: tokio::sync::Mutex<()>, @@ -1787,6 +1801,7 @@ impl BoundaryClient { backend_supports_suspend: AtomicBool::new(false), supervisor_instance_id: crate::boundary_protocol::SupervisorInstanceId::new(), sandbox_bearer, + attach_request_timeout: ATTACH_REQUEST_TIMEOUT, grpc_channel: tokio::sync::Mutex::new(None), mediation: tokio::sync::Mutex::new(None), mediation_open: tokio::sync::Mutex::new(()), @@ -1871,7 +1886,7 @@ impl BoundaryClient { let remember_attach = matches!(request, Request::Attach { .. }); let remember_confirm = matches!(request, Request::Confirm); let timeout = if remember_attach || matches!(request, Request::DiscoverPolicy) { - ATTACH_REQUEST_TIMEOUT + self.attach_request_timeout } else { REQUEST_TIMEOUT }; @@ -2457,6 +2472,7 @@ impl BoundaryClient { ) -> Result<(tonic::transport::Channel, Arc), BackendError> { let runtime_descriptor = self.runtime_descriptor.clone(); let source = self.transport.clone(); + let initial_connect_timeout = self.attach_request_timeout; let transport = Arc::new(TransportAbort::default()); let connector_transport = transport.clone(); let authority = match &runtime_descriptor.transport { @@ -2490,7 +2506,7 @@ impl BoundaryClient { return Err(AbortableTransport::aborted_error()); } source - .connect(&runtime_descriptor) + .connect(&runtime_descriptor, initial_connect_timeout) .await .map(|inner| TokioIo::new(AbortableTransport { inner, abort })) .map_err(|error| std::io::Error::other(error.to_string())) @@ -3044,6 +3060,7 @@ mod tests { use crate::isolation_protocol::exchange_frame; use rustls::pki_types::PrivateKeyDer; + use std::future::{Future as _, poll_fn}; use std::path::PathBuf; use std::pin::Pin; use std::task::{Context, Poll}; @@ -3493,6 +3510,7 @@ mod tests { tls_runtime_descriptor(address, client_tls), test_bearer(&"a".repeat(32)), BoundaryTransport::default(), + ATTACH_REQUEST_TIMEOUT, ) .await; assert!( @@ -3536,6 +3554,7 @@ mod tests { tls_runtime_descriptor(address, client_tls), test_bearer(&"a".repeat(32)), BoundaryTransport::default(), + ATTACH_REQUEST_TIMEOUT, ) .await; assert!( @@ -4160,6 +4179,123 @@ mod tests { server_config: Arc, } + #[tokio::test(start_paused = true)] + async fn configured_initial_timeout_bounds_attach_and_discovery() { + for attach in [false, true] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let descriptor = tls_runtime_descriptor( + listener.local_addr().unwrap(), + test_certificate().client_tls, + ); + let mut client = BoundaryClient::new(descriptor, test_bearer(&"a".repeat(32))); + client.transport = + BoundaryTransport::connector(Arc::new(ReverseTestConnector(listener))); + client.attach_request_timeout = Duration::from_secs(2); + let request = if attach { + Request::Attach { + supervisor_instance_id: client.supervisor_instance_id, + policy: Box::new(SandboxPolicyWire::from(sandbox().policy)), + supervisor_takeover: None, + supervisor_recovery_timeout_secs: None, + } + } else { + Request::DiscoverPolicy + }; + let mut waiting = Box::pin(client.call_idempotent(request)); + poll_fn(|context| { + assert!(waiting.as_mut().poll(context).is_pending()); + Poll::Ready(()) + }) + .await; + tokio::time::advance(Duration::from_secs(2)).await; + let result = poll_fn(|context| { + let Poll::Ready(result) = waiting.as_mut().poll(context) else { + panic!("initial request must finish at its configured two-second deadline"); + }; + Poll::Ready(result) + }) + .await; + assert!(matches!(result, Err(BackendError::Unavailable(message)) + if message.contains("idempotent control request timed out"))); + } + } + + #[tokio::test(start_paused = true)] + async fn configured_routed_bootstrap_wait_preserves_reconnect_window() { + let certificate = test_certificate(); + let mut descriptor = + tls_runtime_descriptor("127.0.0.1:1".parse().unwrap(), certificate.client_tls); + let router = + crate::runtime_routed::RuntimeRouter::bind("127.0.0.1:0".parse().unwrap()).unwrap(); + let route = crate::runtime_routed::new_route_handle(); + descriptor.transport = SandboxTransport::RuntimeInitiatedRouted { + authority: "sandbox.test".to_string(), + route: route.clone(), + }; + let registration = router.register(&route, &descriptor.tls).unwrap(); + let transport = BoundaryTransport::routed(&descriptor, registration.clone()).unwrap(); + let budget = Duration::from_mins(15); + let mut waiting = Box::pin(transport.connect(&descriptor, budget)); + poll_fn(|context| { + assert!(waiting.as_mut().poll(context).is_pending()); + Poll::Ready(()) + }) + .await; + tokio::time::advance(Duration::from_mins(6)).await; + poll_fn(|context| { + assert!( + waiting.as_mut().poll(context).is_pending(), + "configured bootstrap survives the old five-minute ceiling" + ); + Poll::Ready(()) + }) + .await; + // Real socket/TLS progress uses a normal clock; virtual time is only + // needed to cross the long bootstrap and recovery deadlines. + tokio::time::resume(); + let address = router.local_addr(); + let runtime = tokio::spawn(async move { + let mut stream = tokio::net::TcpStream::connect(address).await.unwrap(); + stream + .write_all(&crate::runtime_routed::encode_route_preface(&route).unwrap()) + .await + .unwrap(); + tokio_rustls::TlsAcceptor::from(certificate.server_config) + .accept(stream) + .await + .unwrap() + }); + let stream = tokio::time::timeout(Duration::from_secs(5), &mut waiting) + .await + .unwrap() + .unwrap(); + drop(waiting); + assert!(registration.has_connected()); + drop(stream); + drop(runtime.await.unwrap()); + + tokio::time::pause(); + let mut reconnect = Box::pin(transport.connect(&descriptor, budget)); + poll_fn(|context| { + assert!(reconnect.as_mut().poll(context).is_pending()); + Poll::Ready(()) + }) + .await; + tokio::time::advance(Duration::from_secs(29)).await; + poll_fn(|context| { + assert!(reconnect.as_mut().poll(context).is_pending()); + Poll::Ready(()) + }) + .await; + tokio::time::advance(Duration::from_secs(1)).await; + // Resuming real I/O can leave the paused clock between Tokio's 1 ms timer ticks. + let result = tokio::time::timeout(Duration::from_millis(1), &mut reconnect) + .await + .expect("routed reconnect must finish within one timer tick after thirty seconds"); + assert!(matches!(result, Err(BackendError::Unavailable(message)) + if message.contains("timed out waiting for the sandbox runtime to connect"))); + } + fn test_certificate() -> TestCertificate { test_certificate_with_protocol_versions(&[&rustls::version::TLS13]) } diff --git a/crates/openshell-sandbox-backend/src/runtime/tests/delegated.rs b/crates/openshell-sandbox-backend/src/runtime/tests/delegated.rs index 6275eff8bd..53360ea8d5 100644 --- a/crates/openshell-sandbox-backend/src/runtime/tests/delegated.rs +++ b/crates/openshell-sandbox-backend/src/runtime/tests/delegated.rs @@ -59,7 +59,8 @@ async fn external_backend( EXTERNAL_NAME.into(), descriptor.clone(), OPAQUE_PAYLOAD.to_vec(), - bearer.clone() + bearer.clone(), + ATTACH_REQUEST_TIMEOUT ) .await .unwrap(), @@ -168,7 +169,8 @@ async fn delegated_backend_rejects_reserved_name_and_changed_payload_before_io() crate::BACKEND_NAME.into(), descriptor, OPAQUE_PAYLOAD.to_vec(), - test_bearer(&"a".repeat(32)) + test_bearer(&"a".repeat(32)), + ATTACH_REQUEST_TIMEOUT ) .await, Err(BackendError::Descriptor(_)) diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 1f0f215881..702878c68d 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -705,6 +705,27 @@ fn prepare_server_config_with_drivers( config.image_preparation_timeout_seconds = seconds; } + if let Some(raw) = std::env::var_os("OPENSHELL_PROVISIONING_REPAIR_WINDOW_MS") { + let raw = raw.to_str().ok_or_else(|| { + miette::miette!( + "OPENSHELL_PROVISIONING_REPAIR_WINDOW_MS must be an integer number of milliseconds" + ) + })?; + let window = raw.parse::().map_err(|_| { + miette::miette!( + "OPENSHELL_PROVISIONING_REPAIR_WINDOW_MS must be an integer number of milliseconds" + ) + })?; + validate_provisioning_repair_window_ms(window)?; + config.provisioning_repair_window_ms = window; + } else if let Some(window) = file + .as_ref() + .and_then(|f| f.openshell.gateway.provisioning_repair_window_ms) + { + validate_provisioning_repair_window_ms(window)?; + config.provisioning_repair_window_ms = window; + } + if let Some(issuer) = args.oidc_issuer.clone() { config = config.with_oidc(openshell_core::OidcConfig { issuer, @@ -1620,6 +1641,15 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches: } } +fn validate_provisioning_repair_window_ms(window: i64) -> Result<()> { + if !(1_000..=86_400_000).contains(&window) { + return Err(miette::miette!( + "provisioning_repair_window_ms must be between 1000 and 86400000" + )); + } + Ok(()) +} + fn validate_grpc_rate_limit_args(requests: Option, window_seconds: Option) -> Result<()> { let disabled = matches!(requests, Some(0)) || matches!(window_seconds, Some(0)); if disabled { @@ -4941,6 +4971,112 @@ mem_mib = "not-a-number" }); } + #[test] + fn server_config_prepares_provisioning_repair_window_from_toml() { + let state = tempfile::tempdir().unwrap(); + let tls = tempfile::tempdir().unwrap(); + Environment::new() + .set("XDG_STATE_HOME", state.path().to_str().unwrap()) + .set("OPENSHELL_LOCAL_TLS_DIR", tls.path().to_str().unwrap()) + .run(|| { + let state = + std::path::PathBuf::from(std::env::var_os("XDG_STATE_HOME").unwrap()); + let config_path = state.join("gateway.toml"); + std::fs::write( + &config_path, + "[openshell]\nversion = 2\n[openshell.gateway]\nprovisioning_repair_window_ms = 900000\n", + ) + .unwrap(); + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--config", + config_path.to_str().unwrap(), + "--db-url", + "sqlite::memory:", + "--compute-driver", + "podman", + "--disable-tls", + ]); + let prepared = super::prepare_server_config(&mut args, &matches) + .expect("server config is prepared"); + assert_eq!(prepared.config.provisioning_repair_window_ms, 900_000); + }); + } + + #[test] + fn server_config_rejects_unbounded_provisioning_repair_window() { + let state = tempfile::tempdir().unwrap(); + let tls = tempfile::tempdir().unwrap(); + Environment::new() + .set("XDG_STATE_HOME", state.path().to_str().unwrap()) + .set("OPENSHELL_LOCAL_TLS_DIR", tls.path().to_str().unwrap()) + .run(|| { + let state = + std::path::PathBuf::from(std::env::var_os("XDG_STATE_HOME").unwrap()); + let config_path = state.join("gateway.toml"); + for window in [999, 86_400_001] { + std::fs::write( + &config_path, + format!( + "[openshell]\nversion = 2\n[openshell.gateway]\nprovisioning_repair_window_ms = {window}\n" + ), + ) + .unwrap(); + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--config", + config_path.to_str().unwrap(), + "--db-url", + "sqlite::memory:", + "--compute-driver", + "podman", + "--disable-tls", + ]); + let Err(error) = super::prepare_server_config(&mut args, &matches) else { + panic!("unbounded repair window must be rejected"); + }; + assert!( + error + .to_string() + .contains("provisioning_repair_window_ms must be between 1000 and 86400000") + ); + } + }); + } + + #[test] + fn server_config_env_overrides_provisioning_repair_window() { + let state = tempfile::tempdir().unwrap(); + let tls = tempfile::tempdir().unwrap(); + Environment::new() + .set("XDG_STATE_HOME", state.path().to_str().unwrap()) + .set("OPENSHELL_LOCAL_TLS_DIR", tls.path().to_str().unwrap()) + .set("OPENSHELL_PROVISIONING_REPAIR_WINDOW_MS", "600000") + .run(|| { + let state = + std::path::PathBuf::from(std::env::var_os("XDG_STATE_HOME").unwrap()); + let config_path = state.join("gateway.toml"); + std::fs::write( + &config_path, + "[openshell]\nversion = 2\n[openshell.gateway]\nprovisioning_repair_window_ms = 900000\n", + ) + .unwrap(); + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--config", + config_path.to_str().unwrap(), + "--db-url", + "sqlite::memory:", + "--compute-driver", + "podman", + "--disable-tls", + ]); + let prepared = super::prepare_server_config(&mut args, &matches) + .expect("server config is prepared"); + assert_eq!(prepared.config.provisioning_repair_window_ms, 600_000); + }); + } + #[test] fn explicit_launch_signing_config_ignores_partial_local_bundle() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index a6768040e6..87a46ced30 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -937,6 +937,7 @@ pub struct ComputeRuntime { driver_process: Option>, default_image: String, image_preparation_timeout_seconds: u32, + provisioning_repair_window_ms: i64, sandbox_updates_enabled: bool, update_recovery_sweep: Arc, update_recovery_backoff: Arc>>, @@ -1059,6 +1060,7 @@ impl ComputeRuntime { driver_process, default_image, image_preparation_timeout_seconds: 1800, + provisioning_repair_window_ms: 300_000, sandbox_updates_enabled: false, update_recovery_sweep: Arc::new(tokio::sync::Semaphore::new(1)), update_recovery_backoff: Arc::default(), @@ -1342,6 +1344,20 @@ impl ComputeRuntime { Ok(self) } + /// Validate the repair window once at startup. Persisted attempts keep + /// their original deadline even if the operator changes this value on + /// restart. + pub(crate) fn with_provisioning_repair_window_ms( + mut self, + window: i64, + ) -> Result { + if !(1_000..=86_400_000).contains(&window) { + return Err("provisioning_repair_window_ms must be between 1000 and 86400000".into()); + } + self.provisioning_repair_window_ms = window; + Ok(self) + } + #[must_use] pub fn driver_info_snapshots(&self) -> &[ComputeDriverInfoSnapshot] { std::slice::from_ref(&self.driver_info) @@ -2522,6 +2538,7 @@ impl ComputeRuntime { reason, message, self.image_preparation_timeout_seconds, + self.provisioning_repair_window_ms, ); if let Some(metadata) = sandbox.metadata.as_mut() { next_identity_for_update.write(&mut metadata.annotations); @@ -2954,6 +2971,7 @@ impl ComputeRuntime { "ResumeRolledBack", "Resumed workload stopped after its runtime binding could not be committed", self.image_preparation_timeout_seconds, + self.provisioning_repair_window_ms, ); } if self.restore_lifecycle_snapshot(&settled, &rollback).await { @@ -3031,6 +3049,7 @@ impl ComputeRuntime { "Suspended", "Sandbox remains suspended after failed resume", self.image_preparation_timeout_seconds, + self.provisioning_repair_window_ms, ); let _ = self .restore_lifecycle_snapshot(transition, &suspended) @@ -3171,6 +3190,7 @@ impl ComputeRuntime { &reason, &message, self.image_preparation_timeout_seconds, + self.provisioning_repair_window_ms, ); }, ) @@ -5958,7 +5978,9 @@ impl ComputeRuntime { return Err("restarting sandbox requires a fresh supervisor instance".into()); } let provisioning = admission - .map(|admission| admitted_provisioning(¤t, admission)) + .map(|admission| { + admitted_provisioning(¤t, admission, self.provisioning_repair_window_ms) + }) .transpose()? .flatten(); let expected_resource_version = sandbox_resource_version(¤t); @@ -7861,6 +7883,7 @@ fn record_session_admission( fn admitted_provisioning( sandbox: &Sandbox, admission: &openshell_core::proto::SandboxConfigurationAdmission, + repair_window_ms: i64, ) -> Result, String> { let Some(mut record) = sandbox .status @@ -7873,9 +7896,9 @@ fn admitted_provisioning( if !provisioning_deadline::allows_admission(&record, now_ms) { return Err("provisioning deadline expired".to_string()); } - provisioning_deadline::record_admission_start(&mut record, now_ms)?; + provisioning_deadline::record_admission_start(&mut record, now_ms, repair_window_ms)?; if admission.state == i32::from(openshell_core::proto::ConfigurationAdmissionState::Rejected) { - provisioning_deadline::record_rejection(&mut record, now_ms)?; + provisioning_deadline::record_rejection(&mut record, now_ms, repair_window_ms)?; } Ok(Some(record)) } @@ -8229,6 +8252,7 @@ fn apply_lifecycle_phase( reason: &str, message: &str, image_preparation_timeout_seconds: u32, + provisioning_repair_window_ms: i64, ) { let preserve_exit_code = phase == SandboxPhase::Stopping && (sandbox.phase() == SandboxPhase::Completed as i32 @@ -8260,6 +8284,7 @@ fn apply_lifecycle_phase( status.provisioning = Some(provisioning_deadline::new_preparation_record( openshell_core::time::now_ms(), image_preparation_timeout_seconds, + provisioning_repair_window_ms, )); status.configuration_admission = Some(openshell_core::proto::SandboxConfigurationAdmission { @@ -8726,6 +8751,7 @@ pub fn new_test_runtime_with_driver( driver_process: None, default_image: "openshell/sandbox:test".to_string(), image_preparation_timeout_seconds: 1800, + provisioning_repair_window_ms: 300_000, sandbox_updates_enabled: false, update_recovery_sweep: Arc::new(tokio::sync::Semaphore::new(1)), update_recovery_backoff: Arc::default(), @@ -10024,6 +10050,7 @@ mod tests { driver_process: None, default_image: "openshell/sandbox:test".to_string(), image_preparation_timeout_seconds: 1800, + provisioning_repair_window_ms: 300_000, sandbox_updates_enabled: false, update_recovery_sweep: Arc::new(Semaphore::new(1)), update_recovery_backoff: Arc::default(), @@ -13216,6 +13243,7 @@ mod tests { provisioning_deadline::record_admission_start( status.provisioning.as_mut().unwrap(), openshell_core::time::now_ms(), + runtime.provisioning_repair_window_ms, ) .unwrap(); status.configuration_admission.as_mut().unwrap().state = @@ -19258,7 +19286,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" async fn seed_provisioning_attempt(runtime: &ComputeRuntime) -> Sandbox { let mut sandbox = sandbox_record("sb-ttl", "sandbox-ttl", SandboxPhase::Provisioning); let status = sandbox.status.as_mut().unwrap(); - status.provisioning = Some(provisioning_deadline::new_record(0)); + status.provisioning = Some(provisioning_deadline::new_record(0, 300_000)); status.configuration_activated = Some(false); status.configuration_admission = Some(openshell_core::proto::SandboxConfigurationAdmission { @@ -19345,9 +19373,12 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" "result-owner", SandboxPhase::Provisioning, ); - sandbox.status.as_mut().unwrap().provisioning = Some( - provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 1800), - ); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record( + openshell_core::time::now_ms(), + 1800, + 300_000, + )); let creating = runtime.clone(); let create = tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); @@ -19439,8 +19470,9 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" other.lifecycle_gates = Arc::new(LifecycleGateRegistry::default()); let now = openshell_core::time::now_ms(); let mut sandbox = sandbox_record("sb-stop-lease", "stop-lease", SandboxPhase::Provisioning); - sandbox.status.as_mut().unwrap().provisioning = - Some(provisioning_deadline::new_preparation_record(now, 1)); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(now, 1, 300_000), + ); let creating = runtime.clone(); let create = tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); @@ -19537,6 +19569,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" status.provisioning = Some(provisioning_deadline::new_preparation_record( openshell_core::time::now_ms(), 1800, + 300_000, )); let creating = runtime.clone(); let create = @@ -19643,6 +19676,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" status.main_process_instance_id = "instance-1".into(); status.provisioning = Some(provisioning_deadline::new_record( openshell_core::time::now_ms(), + 300_000, )); let old_generation = sandbox_runtime_generation(&sandbox).unwrap(); runtime.store.put_message(&sandbox).await.unwrap(); @@ -19757,8 +19791,11 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" let driver = ControlledDriver::new(); let runtime = test_runtime(driver.clone()).await; let mut previous = sandbox_record("sb-start-owner", "start-owner", SandboxPhase::Stopped); - let mut record = - provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 1800); + let mut record = provisioning_deadline::new_preparation_record( + openshell_core::time::now_ms(), + 1800, + 300_000, + ); record.driver_operation_id = "old-operation".into(); previous.status.as_mut().unwrap().provisioning = Some(record); runtime.store.put_message(&previous).await.unwrap(); @@ -19829,7 +19866,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" SandboxPhase::Provisioning, ); sandbox.status.as_mut().unwrap().provisioning = Some( - provisioning_deadline::new_record(openshell_core::time::now_ms()), + provisioning_deadline::new_record(openshell_core::time::now_ms(), 300_000), ); runtime.store.put_message(&sandbox).await.unwrap(); let owned = runtime @@ -19908,6 +19945,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" status.main_process_instance_id = "instance-1".into(); status.provisioning = Some(provisioning_deadline::new_record( openshell_core::time::now_ms(), + 300_000, )); runtime.store.put_message(&sandbox).await.unwrap(); runtime @@ -19968,8 +20006,9 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" let pool = sqlx::SqlitePool::connect(&database_url).await.unwrap(); let now = openshell_core::time::now_ms(); let mut sandbox = sandbox_record("sb-monitor", "monitor", SandboxPhase::Provisioning); - sandbox.status.as_mut().unwrap().provisioning = - Some(provisioning_deadline::new_preparation_record(now, 60)); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(now, 60, 300_000), + ); let creating = runtime.clone(); let create = tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); @@ -20021,8 +20060,9 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" let mut runtime = test_runtime(driver.clone()).await; let now = openshell_core::time::now_ms(); let mut sandbox = sandbox_record("sb-owned", "owned", SandboxPhase::Provisioning); - sandbox.status.as_mut().unwrap().provisioning = - Some(provisioning_deadline::new_preparation_record(now, 60)); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(now, 60, 300_000), + ); let root = tempfile::tempdir().unwrap(); let upload = stage_create_test_upload(&mut runtime, &mut sandbox, root.path()); let creating = runtime.clone(); @@ -20101,9 +20141,12 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" driver.set_delete_outcome(ControlledDeleteOutcome::Ok(false)); let mut runtime = test_runtime(driver).await; let mut sandbox = sandbox_record("sb-rejected", "rejected", SandboxPhase::Provisioning); - sandbox.status.as_mut().unwrap().provisioning = Some( - provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 60), - ); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record( + openshell_core::time::now_ms(), + 60, + 300_000, + )); let root = tempfile::tempdir().unwrap(); let upload = stage_create_test_upload(&mut runtime, &mut sandbox, root.path()); let error = runtime @@ -20181,9 +20224,12 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" runtime.store = Arc::new(Store::connect(&database_url).await.unwrap()); let pool = sqlx::SqlitePool::connect(&database_url).await.unwrap(); let mut sandbox = sandbox_record("sb-settlement", "settlement", SandboxPhase::Provisioning); - sandbox.status.as_mut().unwrap().provisioning = Some( - provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 60), - ); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record( + openshell_core::time::now_ms(), + 60, + 300_000, + )); let creating = runtime.clone(); let create = tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); @@ -20241,7 +20287,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" driver.block_create(); let mut runtime = test_runtime(driver.clone()).await; let now = openshell_core::time::now_ms(); - let preparation = provisioning_deadline::new_preparation_record(now, 1); + let preparation = provisioning_deadline::new_preparation_record(now, 1, 300_000); let mut sandbox = sandbox_record("sb-create-ttl", "create-ttl", SandboxPhase::Provisioning); sandbox.status.as_mut().unwrap().provisioning = Some(preparation.clone()); let upload_root = tempfile::tempdir().unwrap(); @@ -20373,8 +20419,9 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" "compensation-ttl", SandboxPhase::Provisioning, ); - sandbox.status.as_mut().unwrap().provisioning = - Some(provisioning_deadline::new_preparation_record(now, 1)); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(now, 1, 300_000), + ); runtime.store.put_message(&sandbox).await.unwrap(); let stale = runtime .store @@ -20423,8 +20470,9 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" "create-retry", SandboxPhase::Provisioning, ); - sandbox.status.as_mut().unwrap().provisioning = - Some(provisioning_deadline::new_preparation_record(now, 1)); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(now, 1, 300_000), + ); let original_attempt = sandbox_provisioning_attempt_id(&sandbox) .unwrap() .to_owned(); @@ -20578,9 +20626,12 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" async fn failed_create_cleanup_preserves_retry_after_cas_conflict() { let runtime = test_runtime(ControlledDriver::new()).await; let mut sandbox = sandbox_record("sb-retry-cas", "retry-cas", SandboxPhase::Provisioning); - sandbox.status.as_mut().unwrap().provisioning = Some( - provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 1), - ); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record( + openshell_core::time::now_ms(), + 1, + 300_000, + )); runtime.store.put_message(&sandbox).await.unwrap(); let stale = runtime .store @@ -20600,6 +20651,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" "Starting", "Sandbox start requested", runtime.image_preparation_timeout_seconds, + runtime.provisioning_repair_window_ms, ); }, ) @@ -20631,9 +20683,12 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" "binding-retry", SandboxPhase::Provisioning, ); - sandbox.status.as_mut().unwrap().provisioning = Some( - provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 1), - ); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record( + openshell_core::time::now_ms(), + 1, + 300_000, + )); runtime.store.put_message(&sandbox).await.unwrap(); let stale = runtime .store @@ -20653,6 +20708,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" "Starting", "Sandbox start requested", runtime.image_preparation_timeout_seconds, + runtime.provisioning_repair_window_ms, ); current.set_phase(SandboxPhase::Ready.into()); set_compute_runtime_binding(current, "retry-runtime"); @@ -20703,7 +20759,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" let mut runtime = test_runtime(driver.clone()).await; runtime.driver_info.supports_sandbox_authentication = true; let now = openshell_core::time::now_ms(); - let preparation = provisioning_deadline::new_preparation_record(now, 1); + let preparation = provisioning_deadline::new_preparation_record(now, 1, 300_000); let mut sandbox = sandbox_record("sb-late-create", "late-create", SandboxPhase::Provisioning); sandbox.status.as_mut().unwrap().provisioning = Some(preparation.clone()); @@ -20776,7 +20832,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" driver.block_start(); let runtime = test_runtime(driver.clone()).await; let now = openshell_core::time::now_ms(); - let preparation = provisioning_deadline::new_preparation_record(now, 1800); + let preparation = provisioning_deadline::new_preparation_record(now, 1800, 300_000); let mut sandbox = sandbox_record("sb-recovery-ttl", "recovery-ttl", SandboxPhase::Starting); sandbox.status.as_mut().unwrap().provisioning = Some(preparation.clone()); runtime.store.put_message(&sandbox).await.unwrap(); @@ -20859,8 +20915,9 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" let driver = ControlledDriver::new(); let runtime = test_runtime(driver.clone()).await; let mut sandbox = sandbox_record("sb-prepare", "prepare", SandboxPhase::Provisioning); - sandbox.status.as_mut().unwrap().provisioning = - Some(provisioning_deadline::new_preparation_record(0, 1800)); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(0, 1800, 300_000), + ); runtime.store.put_message(&sandbox).await.unwrap(); let sandbox = runtime .store @@ -21445,10 +21502,15 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" let now = openshell_core::time::now_ms(); let mut sandbox = sandbox_record("sb-reset", "reset", SandboxPhase::Provisioning); sandbox.status.as_mut().unwrap().provisioning = - Some(provisioning_deadline::new_record(now - 100_000)); - provisioning_deadline::refresh_configuration(&runtime.store, &mut sandbox, now - 100_000) - .await - .unwrap(); + Some(provisioning_deadline::new_record(now - 100_000, 300_000)); + provisioning_deadline::refresh_configuration( + &runtime.store, + &mut sandbox, + now - 100_000, + 300_000, + ) + .await + .unwrap(); runtime.store.put_message(&sandbox).await.unwrap(); let policy = openshell_policy::restrictive_default_policy(); let hash = openshell_core::policy_identity::deterministic_policy_hash(&policy); @@ -21501,7 +21563,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" let now = openshell_core::time::now_ms(); let mut sandbox = sandbox_record("sb-busy-ttl", "busy-ttl", SandboxPhase::Starting); sandbox.status.as_mut().unwrap().provisioning = - Some(provisioning_deadline::new_record(now)); + Some(provisioning_deadline::new_record(now, 300_000)); runtime.store.put_message(&sandbox).await.unwrap(); let gate = runtime.lifecycle_gates.lock_for("sb-busy-ttl").await; runtime diff --git a/crates/openshell-server/src/compute/provisioning_deadline.rs b/crates/openshell-server/src/compute/provisioning_deadline.rs index cb351e64d0..2763f86d1a 100644 --- a/crates/openshell-server/src/compute/provisioning_deadline.rs +++ b/crates/openshell-server/src/compute/provisioning_deadline.rs @@ -12,8 +12,6 @@ use openshell_core::proto::SandboxProvisioning; use openshell_core::time::{timestamp_from_millis, timestamp_to_millis}; use serde::{Deserialize, Serialize}; -const REPAIR_WINDOW_MS: i64 = 300_000; - /// How long one scan pass waits, in total, for sandboxes' local keys. A sandbox /// whose keys are still busy is skipped until the next pass. Keys are taken in /// ascending order, so the scan can hold X(sandbox) while it waits for @@ -136,7 +134,12 @@ impl ProvisioningDeadline { }; } - pub fn new(attempt_id: String, change: ConfigurationChange, now_ms: i64) -> Self { + pub fn new( + attempt_id: String, + change: ConfigurationChange, + now_ms: i64, + repair_window_ms: i64, + ) -> Self { Self { attempt_id, change, @@ -144,7 +147,7 @@ impl ProvisioningDeadline { preparation_deadline_at_ms: None, admission_start_at_ms: None, state: DeadlineState::Armed { - deadline_at_ms: now_ms.saturating_add(REPAIR_WINDOW_MS), + deadline_at_ms: now_ms.saturating_add(repair_window_ms), }, } } @@ -156,7 +159,7 @@ impl ProvisioningDeadline { /// Only an authenticated report for the current supervisor may call this. /// Persist it with the report: duplicate delivery or restart must not grant /// another admission window, and a late registration cannot revive compute. - fn start_admission(&mut self, attempt_id: &str, now_ms: i64) -> bool { + fn start_admission(&mut self, attempt_id: &str, now_ms: i64, repair_window_ms: i64) -> bool { if attempt_id != self.attempt_id || !self.is_preparing() || now_ms < self.change.committed_at_ms @@ -168,7 +171,7 @@ impl ProvisioningDeadline { } self.admission_start_at_ms = Some(now_ms); self.state = DeadlineState::Armed { - deadline_at_ms: now_ms.saturating_add(REPAIR_WINDOW_MS), + deadline_at_ms: now_ms.saturating_add(repair_window_ms), }; true } @@ -182,7 +185,12 @@ impl ProvisioningDeadline { /// Apply a newer, committed change to an active attempt. Delayed observation /// uses commit time, not poll time. A change after expiry cannot revive it. - pub fn configuration_changed(&mut self, attempt_id: &str, change: ConfigurationChange) -> bool { + pub fn configuration_changed( + &mut self, + attempt_id: &str, + change: ConfigurationChange, + repair_window_ms: i64, + ) -> bool { let Some(deadline_at_ms) = self.deadline_at_ms() else { return false; }; @@ -196,7 +204,7 @@ impl ProvisioningDeadline { if !self.is_preparing() { self.state = DeadlineState::Armed { deadline_at_ms: deadline_at_ms - .max(change.committed_at_ms.saturating_add(REPAIR_WINDOW_MS)), + .max(change.committed_at_ms.saturating_add(repair_window_ms)), }; } self.change = change; @@ -206,7 +214,13 @@ impl ProvisioningDeadline { /// Only the first accepted rejection for the current change grants a repair /// window. Retrying, reconnecting, or changing diagnostic text does not. - pub fn rejected(&mut self, attempt_id: &str, change_id: &str, now_ms: i64) -> bool { + pub fn rejected( + &mut self, + attempt_id: &str, + change_id: &str, + now_ms: i64, + repair_window_ms: i64, + ) -> bool { let Some(deadline_at_ms) = self.deadline_at_ms() else { return false; }; @@ -220,7 +234,7 @@ impl ProvisioningDeadline { } self.first_rejection_at_ms = Some(now_ms); self.state = DeadlineState::Armed { - deadline_at_ms: deadline_at_ms.max(now_ms.saturating_add(REPAIR_WINDOW_MS)), + deadline_at_ms: deadline_at_ms.max(now_ms.saturating_add(repair_window_ms)), }; true } @@ -281,7 +295,7 @@ pub(super) fn driver_operation_pending(sandbox: &openshell_core::proto::Sandbox) /// Adopt an existing untimed attempt without granting it a new preparation phase. /// New create/start operations use `new_preparation_record` instead. -pub fn new_record(now_ms: i64) -> SandboxProvisioning { +pub fn new_record(now_ms: i64, repair_window_ms: i64) -> SandboxProvisioning { let mut record = SandboxProvisioning::default(); ProvisioningDeadline::new( uuid::Uuid::new_v4().to_string(), @@ -290,6 +304,7 @@ pub fn new_record(now_ms: i64) -> SandboxProvisioning { committed_at_ms: now_ms, }, now_ms, + repair_window_ms, ) .write_record(&mut record); record @@ -298,8 +313,12 @@ pub fn new_record(now_ms: i64) -> SandboxProvisioning { /// Create an independent attempt with a fixed preparation budget. The gateway /// validates the configured seconds before constructing the runtime. Reconnects /// and driver progress must never call this function. -pub fn new_preparation_record(now_ms: i64, timeout_seconds: u32) -> SandboxProvisioning { - let mut record = new_record(now_ms); +pub fn new_preparation_record( + now_ms: i64, + timeout_seconds: u32, + repair_window_ms: i64, +) -> SandboxProvisioning { + let mut record = new_record(now_ms, repair_window_ms); let ceiling = now_ms.saturating_add(i64::from(timeout_seconds) * 1_000); record.preparation_deadline = timestamp_from_millis(ceiling).ok(); record.deadline.clone_from(&record.preparation_deadline); @@ -309,18 +328,31 @@ pub fn new_preparation_record(now_ms: i64, timeout_seconds: u32) -> SandboxProvi /// The caller has authenticated the supervisor and checked its instance fence. /// A Pending registration may start timing before configuration validation. /// Persist this transition in the same CAS as the supervisor report. -pub fn record_admission_start(record: &mut SandboxProvisioning, now_ms: i64) -> Result<(), String> { +pub fn record_admission_start( + record: &mut SandboxProvisioning, + now_ms: i64, + repair_window_ms: i64, +) -> Result<(), String> { let mut deadline = ProvisioningDeadline::from_record(record)?; - deadline.start_admission(&record.attempt_id, now_ms); + deadline.start_admission(&record.attempt_id, now_ms, repair_window_ms); deadline.write_record(record); Ok(()) } /// Apply the first accepted rejection under the same CAS as admission evidence. /// The report's generation and supervisor instance must already be validated. -pub fn record_rejection(record: &mut SandboxProvisioning, now_ms: i64) -> Result<(), String> { +pub fn record_rejection( + record: &mut SandboxProvisioning, + now_ms: i64, + repair_window_ms: i64, +) -> Result<(), String> { let mut deadline = ProvisioningDeadline::from_record(record)?; - deadline.rejected(&record.attempt_id, &record.configuration_change_id, now_ms); + deadline.rejected( + &record.attempt_id, + &record.configuration_change_id, + now_ms, + repair_window_ms, + ); deadline.write_record(record); Ok(()) } @@ -401,6 +433,7 @@ pub async fn refresh_configuration( store: &crate::persistence::Store, sandbox: &mut openshell_core::proto::Sandbox, now_ms: i64, + repair_window_ms: i64, ) -> Result<(), String> { use openshell_core::proto::SandboxPhase; if !matches!( @@ -412,7 +445,7 @@ pub async fn refresh_configuration( let status = sandbox.status.get_or_insert_with(Default::default); let record = status .provisioning - .get_or_insert_with(|| new_record(now_ms)); + .get_or_insert_with(|| new_record(now_ms, repair_window_ms)); if record.deadline.is_none() { return Ok(()); } @@ -427,7 +460,7 @@ pub async fn refresh_configuration( // Taking the first source snapshot does not grant a second initial window. deadline.change.id = change.id; } else { - deadline.configuration_changed(&record.attempt_id, change); + deadline.configuration_changed(&record.attempt_id, change, repair_window_ms); } deadline.write_record(record); Ok(()) @@ -518,7 +551,13 @@ impl super::ComputeRuntime { continue; }; let previous = current.clone(); - refresh_configuration(&self.store, &mut current, now_ms).await?; + refresh_configuration( + &self.store, + &mut current, + now_ms, + self.provisioning_repair_window_ms, + ) + .await?; if current != previous { current = self .store @@ -814,6 +853,41 @@ impl super::ComputeRuntime { mod tests { use super::*; + #[test] + fn configured_repair_window_preserves_persisted_deadlines() { + use prost::Message; + let mut record = new_preparation_record(0, 1800, 900_000); + record_admission_start(&mut record, 600_000, 900_000).unwrap(); + assert_eq!( + ProvisioningDeadline::from_record(&record) + .unwrap() + .deadline_at_ms(), + Some(1_500_000) + ); + record_rejection(&mut record, 610_000, 900_000).unwrap(); + let bytes = record.encode_to_vec(); + let mut restored = SandboxProvisioning::decode(bytes.as_slice()).unwrap(); + record_admission_start(&mut restored, 700_000, 1_200_000).unwrap(); + record_rejection(&mut restored, 700_000, 1_200_000).unwrap(); + let mut timer = ProvisioningDeadline::from_record(&restored).unwrap(); + assert_eq!(timer.deadline_at_ms(), Some(1_510_000)); + assert!(!timer.configuration_changed( + "previous-attempt", + change("late", 800_000), + 1_200_000 + )); + assert!(timer.expire( + &restored.attempt_id, + &restored.configuration_change_id, + 1_510_000 + )); + assert!(!timer.configuration_changed( + &restored.attempt_id, + change("after-expiry", 1_500_000), + 1_200_000 + )); + } + #[test] fn existing_wire_record_does_not_gain_preparation_time() { use prost::Message; @@ -824,7 +898,7 @@ mod tests { assert!(!record.driver_operation_pending); assert!(record.driver_operation_id.is_empty()); let before = record.clone(); - record_admission_start(&mut record, 299_999).unwrap(); + record_admission_start(&mut record, 299_999, 300_000).unwrap(); assert_eq!(record, before); assert!(record.preparation_deadline.is_none()); assert!(!allows_admission(&record, 300_000)); @@ -832,18 +906,18 @@ mod tests { #[test] fn preparation_over_five_minutes_gets_a_full_admission_repair_window() { - let record = new_preparation_record(0, 1800); + let record = new_preparation_record(0, 1800, 300_000); let mut timer = ProvisioningDeadline::from_record(&record).unwrap(); let attempt = record.attempt_id; let change_id = record.configuration_change_id; assert!(!timer.expire(&attempt, &change_id, 600_000)); assert!(!timer.ready(&attempt, &change_id, 600_000)); - assert!(timer.start_admission(&attempt, 600_000)); + assert!(timer.start_admission(&attempt, 600_000, 300_000)); assert_eq!(timer.deadline_at_ms(), Some(900_000)); - assert!(timer.rejected(&attempt, &change_id, 601_000)); + assert!(timer.rejected(&attempt, &change_id, 601_000, 300_000)); assert_eq!(timer.deadline_at_ms(), Some(901_000)); - assert!(!timer.start_admission(&attempt, 800_000)); - assert!(timer.configuration_changed(&attempt, change("updated", 800_000))); + assert!(!timer.start_admission(&attempt, 800_000, 300_000)); + assert!(timer.configuration_changed(&attempt, change("updated", 800_000), 300_000)); assert_eq!(timer.deadline_at_ms(), Some(1_100_000)); assert_eq!(timer.admission_start_at_ms, Some(600_000)); assert_eq!(timer.preparation_deadline_at_ms, Some(1_800_000)); @@ -852,15 +926,15 @@ mod tests { #[test] fn preparation_config_changes_and_restart_preserve_the_absolute_ceiling() { use prost::Message; - let mut record = new_preparation_record(0, 1800); + let mut record = new_preparation_record(0, 1800, 300_000); record.driver_operation_pending = true; record.driver_operation_id = "operation-1".into(); let mut timer = ProvisioningDeadline::from_record(&record).unwrap(); let attempt = record.attempt_id.clone(); - assert!(timer.configuration_changed(&attempt, change("first", 600_000))); - assert!(timer.configuration_changed(&attempt, change("last", 1_799_000))); - assert!(!timer.configuration_changed(&attempt, change("last", 1_799_500))); - assert!(!timer.rejected(&attempt, "last", 1_799_500)); + assert!(timer.configuration_changed(&attempt, change("first", 600_000), 300_000)); + assert!(timer.configuration_changed(&attempt, change("last", 1_799_000), 300_000)); + assert!(!timer.configuration_changed(&attempt, change("last", 1_799_500), 300_000)); + assert!(!timer.rejected(&attempt, "last", 1_799_500, 300_000)); timer.write_record(&mut record); let bytes = record.encode_to_vec(); let restored = SandboxProvisioning::decode(bytes.as_slice()).unwrap(); @@ -868,30 +942,30 @@ mod tests { assert_eq!(restored.driver_operation_id, "operation-1"); let mut timer = ProvisioningDeadline::from_record(&restored).unwrap(); assert_eq!(timer.deadline_at_ms(), Some(1_800_000)); - assert!(!timer.start_admission("previous-attempt", 1_799_999)); - assert!(!timer.start_admission(&attempt, 1_800_000)); + assert!(!timer.start_admission("previous-attempt", 1_799_999, 300_000)); + assert!(!timer.start_admission(&attempt, 1_800_000, 300_000)); assert!(timer.expire(&attempt, "last", 1_800_000)); - assert!(!timer.start_admission(&attempt, 1_799_999)); + assert!(!timer.start_admission(&attempt, 1_799_999, 300_000)); assert!(!timer.ready(&attempt, "last", 1_799_999)); - assert!(!timer.configuration_changed(&attempt, change("late", 1_799_999))); + assert!(!timer.configuration_changed(&attempt, change("late", 1_799_999), 300_000)); } #[test] fn admission_registration_roundtrip_does_not_restart_repair() { use prost::Message; - let mut record = new_preparation_record(0, 1800); - record_admission_start(&mut record, 600_000).unwrap(); + let mut record = new_preparation_record(0, 1800, 300_000); + record_admission_start(&mut record, 600_000, 300_000).unwrap(); let before = record.clone(); let bytes = record.encode_to_vec(); let mut restored = SandboxProvisioning::decode(bytes.as_slice()).unwrap(); - record_admission_start(&mut restored, 899_999).unwrap(); + record_admission_start(&mut restored, 899_999, 300_000).unwrap(); assert_eq!(restored, before); assert!(!allows_admission(&restored, 900_000)); } #[test] fn malformed_preparation_timing_cannot_grant_admission() { - let mut record = new_preparation_record(0, 1800); + let mut record = new_preparation_record(0, 1800, 300_000); record.deadline = timestamp_from_millis(1_800_001).ok(); assert!(ProvisioningDeadline::from_record(&record).is_err()); record.deadline = None; @@ -904,8 +978,9 @@ mod tests { #[test] fn protobuf_roundtrip_retains_deadline_and_cleanup_progress() { use prost::Message; - let mut deadline = ProvisioningDeadline::new("attempt".into(), change("change", 0), 0); - assert!(deadline.rejected("attempt", "change", 1_000)); + let mut deadline = + ProvisioningDeadline::new("attempt".into(), change("change", 0), 0, 300_000); + assert!(deadline.rejected("attempt", "change", 1_000, 300_000)); let mut record = SandboxProvisioning::default(); deadline.write_record(&mut record); let bytes = record.encode_to_vec(); @@ -929,7 +1004,7 @@ mod tests { #[test] fn malformed_persisted_timing_cannot_grant_another_window() { assert!(ProvisioningDeadline::from_record(&SandboxProvisioning::default()).is_err()); - let mut record = new_record(0); + let mut record = new_record(0, 300_000); record.timeout_time = timestamp_from_millis(300_000).ok(); assert!(ProvisioningDeadline::from_record(&record).is_err()); record.timeout_time = None; @@ -945,7 +1020,7 @@ mod tests { } fn initial() -> ProvisioningDeadline { - ProvisioningDeadline::new("attempt-1".into(), change("change-1", 0), 0) + ProvisioningDeadline::new("attempt-1".into(), change("change-1", 0), 0, 300_000) } #[test] @@ -959,9 +1034,9 @@ mod tests { #[test] fn repeated_failures_do_not_extend_the_repair_window() { let mut timer = initial(); - assert!(timer.rejected("attempt-1", "change-1", 5_000)); + assert!(timer.rejected("attempt-1", "change-1", 5_000, 300_000)); for now in (7_000..305_000).step_by(2_000) { - assert!(!timer.rejected("attempt-1", "change-1", now)); + assert!(!timer.rejected("attempt-1", "change-1", now, 300_000)); } assert_eq!(timer.deadline_at_ms(), Some(305_000)); assert!(timer.expire("attempt-1", "change-1", 305_000)); @@ -970,12 +1045,12 @@ mod tests { #[test] fn change_and_first_failed_load_each_reset_once() { let mut timer = initial(); - assert!(timer.configuration_changed("attempt-1", change("change-2", 100_000))); + assert!(timer.configuration_changed("attempt-1", change("change-2", 100_000), 300_000)); assert_eq!(timer.deadline_at_ms(), Some(400_000)); - assert!(timer.rejected("attempt-1", "change-2", 105_000)); + assert!(timer.rejected("attempt-1", "change-2", 105_000, 300_000)); assert_eq!(timer.deadline_at_ms(), Some(405_000)); - assert!(!timer.configuration_changed("attempt-1", change("change-2", 110_000))); - assert!(!timer.rejected("attempt-1", "change-2", 110_000)); + assert!(!timer.configuration_changed("attempt-1", change("change-2", 110_000), 300_000)); + assert!(!timer.rejected("attempt-1", "change-2", 110_000, 300_000)); assert_eq!(timer.deadline_at_ms(), Some(405_000)); } @@ -983,17 +1058,17 @@ mod tests { fn delayed_observation_uses_commit_time() { let mut timer = initial(); // Caller can observe this later, but it committed before the deadline. - assert!(timer.configuration_changed("attempt-1", change("change-2", 100_000))); + assert!(timer.configuration_changed("attempt-1", change("change-2", 100_000), 300_000)); assert_eq!(timer.deadline_at_ms(), Some(400_000)); - assert!(!timer.configuration_changed("attempt-1", change("older", 50_000))); + assert!(!timer.configuration_changed("attempt-1", change("older", 50_000), 300_000)); } #[test] fn stale_attempts_and_changes_cannot_extend_or_expire() { let mut timer = initial(); - assert!(!timer.rejected("old-attempt", "change-1", 1_000)); - assert!(!timer.rejected("attempt-1", "old-change", 1_000)); - assert!(!timer.configuration_changed("old-attempt", change("change-2", 1_000))); + assert!(!timer.rejected("old-attempt", "change-1", 1_000, 300_000)); + assert!(!timer.rejected("attempt-1", "old-change", 1_000, 300_000)); + assert!(!timer.configuration_changed("old-attempt", change("change-2", 1_000), 300_000)); assert!(!timer.expire("old-attempt", "change-1", 400_000)); assert!(!timer.expire("attempt-1", "old-change", 400_000)); assert_eq!(timer.deadline_at_ms(), Some(300_000)); @@ -1002,45 +1077,50 @@ mod tests { #[test] fn late_change_failure_or_ready_cannot_escape_expiry() { let mut timer = initial(); - assert!(!timer.configuration_changed("attempt-1", change("change-2", 300_000))); - assert!(!timer.rejected("attempt-1", "change-1", 300_000)); + assert!(!timer.configuration_changed("attempt-1", change("change-2", 300_000), 300_000)); + assert!(!timer.rejected("attempt-1", "change-1", 300_000, 300_000)); assert!(!timer.ready("attempt-1", "change-1", 300_000)); assert!(timer.expire("attempt-1", "change-1", 300_000)); assert!(!timer.ready("attempt-1", "change-1", 300_001)); - assert!(!timer.configuration_changed("attempt-1", change("change-2", 100_000))); + assert!(!timer.configuration_changed("attempt-1", change("change-2", 100_000), 300_000)); } #[test] fn ready_disarms_and_cannot_rearm_for_a_failed_live_update() { let mut timer = initial(); assert!(timer.ready("attempt-1", "change-1", 50_000)); - assert!(!timer.rejected("attempt-1", "change-1", 60_000)); - assert!(!timer.configuration_changed("attempt-1", change("change-2", 60_000))); + assert!(!timer.rejected("attempt-1", "change-1", 60_000, 300_000)); + assert!(!timer.configuration_changed("attempt-1", change("change-2", 60_000), 300_000)); assert!(!timer.expire("attempt-1", "change-1", 400_000)); } #[test] fn restart_roundtrip_preserves_first_failure_and_deadline() { let mut timer = initial(); - assert!(timer.rejected("attempt-1", "change-1", 5_000)); + assert!(timer.rejected("attempt-1", "change-1", 5_000, 300_000)); let encoded = serde_json::to_vec(&timer).unwrap(); let mut restored: ProvisioningDeadline = serde_json::from_slice(&encoded).unwrap(); assert_eq!(restored, timer); - assert!(!restored.rejected("attempt-1", "change-1", 100_000)); + assert!(!restored.rejected("attempt-1", "change-1", 100_000, 300_000)); assert_eq!(restored.deadline_at_ms(), Some(305_000)); } #[test] fn explicit_retry_is_a_new_attempt_and_old_cleanup_cannot_expire_it() { - let mut timer = - ProvisioningDeadline::new("attempt-2".into(), change("change-2", 350_000), 400_000); + let mut timer = ProvisioningDeadline::new( + "attempt-2".into(), + change("change-2", 350_000), + 400_000, + 300_000, + ); assert_eq!(timer.deadline_at_ms(), Some(700_000)); assert!(!timer.expire("attempt-1", "change-2", 800_000)); } #[test] fn legacy_initialization_grants_one_window_from_adoption() { - let timer = ProvisioningDeadline::new("adopted".into(), change("legacy", 0), 1_000_000); + let timer = + ProvisioningDeadline::new("adopted".into(), change("legacy", 0), 1_000_000, 300_000); assert_eq!(timer.deadline_at_ms(), Some(1_300_000)); } } diff --git a/crates/openshell-server/src/compute/tests/error_recovery.rs b/crates/openshell-server/src/compute/tests/error_recovery.rs index 8d269d1804..89a58632ea 100644 --- a/crates/openshell-server/src/compute/tests/error_recovery.rs +++ b/crates/openshell-server/src/compute/tests/error_recovery.rs @@ -371,7 +371,7 @@ async fn pending_driver_operation_blocks_error_recovery_without_mutation() { let driver = ControlledDriver::new(); let runtime = test_runtime(driver.clone()).await; let mut sandbox = error_sandbox_record("pending-id", "pending", "StartFailed"); - let mut operation = provisioning_deadline::new_record(openshell_core::time::now_ms()); + let mut operation = provisioning_deadline::new_record(openshell_core::time::now_ms(), 300_000); // Even a completion timestamp cannot override an unsettled original driver // operation. This also fences malformed/stale timeout cleanup evidence. operation.timeout_time = openshell_core::time::timestamp_from_millis(300_000).ok(); @@ -401,7 +401,7 @@ async fn timeout_stop_preserves_cleanup_and_direct_start_recovery() { driver.track_compute.store(true, Ordering::SeqCst); let runtime = test_runtime(driver.clone()).await; let mut sandbox = error_sandbox_record("timeout-id", "timeout", "ProvisioningTimedOut"); - let mut operation = provisioning_deadline::new_record(0); + let mut operation = provisioning_deadline::new_record(0, 300_000); operation.timeout_time = openshell_core::time::timestamp_from_millis(300_000).ok(); operation.cleanup_retry_time = openshell_core::time::timestamp_from_millis(310_000).ok(); if cleanup_complete { @@ -474,7 +474,7 @@ async fn reclaimed_timeout_can_still_start_directly_without_stop() { "direct-timeout", "ImagePreparationTimedOut", ); - let mut operation = provisioning_deadline::new_record(0); + let mut operation = provisioning_deadline::new_record(0, 300_000); operation.timeout_time = openshell_core::time::timestamp_from_millis(300_000).ok(); operation.cleanup_completed_time = openshell_core::time::timestamp_from_millis(320_000).ok(); sandbox.status.as_mut().unwrap().provisioning = Some(operation); diff --git a/crates/openshell-server/src/compute/tests/suspend_resume.rs b/crates/openshell-server/src/compute/tests/suspend_resume.rs index 0e59e4ec90..395a663493 100644 --- a/crates/openshell-server/src/compute/tests/suspend_resume.rs +++ b/crates/openshell-server/src/compute/tests/suspend_resume.rs @@ -685,9 +685,12 @@ async fn stale_resume_compensation_cannot_clear_a_newer_operation_intent() { let driver = ControlledDriver::new(); let runtime = suspend_capable_runtime(driver.clone()).await; let mut sandbox = sandbox_record("sb-resume-owner", "resume-owner", SandboxPhase::Starting); - sandbox.status.as_mut().unwrap().provisioning = Some( - provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 1800), - ); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record( + openshell_core::time::now_ms(), + 1800, + 300_000, + )); let generation = sandbox_runtime_generation(&sandbox).unwrap(); write_start_intent( &mut sandbox.metadata.as_mut().unwrap().annotations, diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 865f6b1ed1..ce3f2838fd 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -149,6 +149,9 @@ pub struct GatewayFileSection { /// Absolute preparation budget for new attempts; existing deadlines persist. #[serde(default)] pub image_preparation_timeout_seconds: Option, + /// Admission repair window in milliseconds, between 1000 and 86400000. + #[serde(default)] + pub provisioning_repair_window_ms: Option, /// Explicit activation only after the compatible deployment rollout ends. #[serde(default)] pub enable_sandbox_updates: Option, diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index f5c7c94480..604dd38112 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -5103,6 +5103,7 @@ pub(super) async fn handle_report_sandbox_configuration( &state.store, &mut sandbox, current_time_ms(), + state.config.provisioning_repair_window_ms, ) .await .map_err(Status::internal)?; @@ -5196,11 +5197,19 @@ pub(super) async fn handle_report_sandbox_configuration( .map_err(Status::internal)?; return Err(Status::failed_precondition("provisioning deadline expired")); } - crate::compute::provisioning_deadline::record_admission_start(record, now_ms) - .map_err(Status::internal)?; + crate::compute::provisioning_deadline::record_admission_start( + record, + now_ms, + state.config.provisioning_repair_window_ms, + ) + .map_err(Status::internal)?; if reported == ConfigurationAdmissionState::Rejected { - crate::compute::provisioning_deadline::record_rejection(record, now_ms) - .map_err(Status::internal)?; + crate::compute::provisioning_deadline::record_rejection( + record, + now_ms, + state.config.provisioning_repair_window_ms, + ) + .map_err(Status::internal)?; } } let updated = state @@ -8664,7 +8673,7 @@ mod tests { Vec::new(), ); sandbox.set_phase(SandboxPhase::Provisioning.into()); - let preparation = new_preparation_record(current_time_ms() - 600_000, 1800); + let preparation = new_preparation_record(current_time_ms() - 600_000, 1800, 300_000); sandbox.status.as_mut().unwrap().provisioning = Some(preparation.clone()); state.store.put_message(&sandbox).await.unwrap(); let instance_id = uuid::Uuid::new_v4().to_string(); @@ -8753,7 +8762,8 @@ mod tests { Vec::new(), ); sandbox.set_phase(SandboxPhase::Provisioning.into()); - sandbox.status.as_mut().unwrap().provisioning = Some(new_preparation_record(0, 1800)); + sandbox.status.as_mut().unwrap().provisioning = + Some(new_preparation_record(0, 1800, 300_000)); state.store.put_message(&sandbox).await.unwrap(); let error = handle_report_sandbox_configuration( &state, diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 86ff12b170..80d857612e 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -741,12 +741,14 @@ async fn handle_create_sandbox_inner( crate::compute::provisioning_deadline::new_preparation_record( now_ms, state.config.image_preparation_timeout_seconds, + state.config.provisioning_repair_window_ms, ), ); crate::compute::provisioning_deadline::refresh_configuration( &state.store, &mut sandbox, now_ms, + state.config.provisioning_repair_window_ms, ) .await .map_err(Status::internal)?; diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 02c1ba16b3..e5fa3c1986 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -2284,6 +2284,9 @@ async fn build_compute_runtime( .and_then(|runtime| { runtime.with_image_preparation_timeout(config.image_preparation_timeout_seconds) }) + .and_then(|runtime| { + runtime.with_provisioning_repair_window_ms(config.provisioning_repair_window_ms) + }) .map_err(Error::config)?; Ok(runtime.with_telemetry_compute_driver(telemetry_compute_driver)) } diff --git a/crates/openshell-server/src/persistence/sandbox_image.rs b/crates/openshell-server/src/persistence/sandbox_image.rs index 386fe78136..70877c4d52 100644 --- a/crates/openshell-server/src/persistence/sandbox_image.rs +++ b/crates/openshell-server/src/persistence/sandbox_image.rs @@ -854,7 +854,8 @@ mod tests { #[test] fn image_completion_preserves_concurrent_attachment_clock_edits() { let mut original = sandbox(); - let mut previous = crate::compute::provisioning_deadline::new_preparation_record(0, 1800); + let mut previous = + crate::compute::provisioning_deadline::new_preparation_record(0, 1800, 300_000); previous.driver_operation_id = "prior-operation".into(); previous.attachment_change_id = "before-attachment".into(); original.status.as_mut().unwrap().provisioning = Some(previous); diff --git a/crates/openshell-supervisor/src/backend_setup.rs b/crates/openshell-supervisor/src/backend_setup.rs index a0cd52ea87..eacec32962 100644 --- a/crates/openshell-supervisor/src/backend_setup.rs +++ b/crates/openshell-supervisor/src/backend_setup.rs @@ -38,6 +38,7 @@ pub struct BackendServices { pub ca_file_paths: Arc>>, pub provider_credentials: ProviderCredentialState, pub sandbox_bearer: SessionBearerTokenSlot, + pub attach_request_timeout: std::time::Duration, /// Gateway-granted authority to replace the generation's current owner. pub takeover: Option, /// Recovery deadline (seconds) offered to the runtime at every attach. @@ -75,6 +76,7 @@ pub trait PreparedBackend: Send + Sync { async fn discover_policy( &self, bearer: SessionBearerTokenSlot, + attach_request_timeout: std::time::Duration, ) -> std::result::Result<(Option, bool), BackendError>; /// Consume prepared launch data and retain the supervisor's live handles. @@ -159,9 +161,10 @@ impl SelectedBackend { pub async fn discover_policy( &self, bearer: SessionBearerTokenSlot, + attach_request_timeout: std::time::Duration, ) -> Result<(Option, bool)> { self.prepared - .discover_policy(bearer) + .discover_policy(bearer, attach_request_timeout) .await .map_err(|error| miette::miette!("discover workload image policy: {error}")) } @@ -314,11 +317,13 @@ impl PreparedBackend for OpenShellLaunch { async fn discover_policy( &self, bearer: SessionBearerTokenSlot, + attach_request_timeout: std::time::Duration, ) -> std::result::Result<(Option, bool), BackendError> { openshell_sandbox_backend::OpenShellRuntimeBackend::discover_policy( self.descriptor.clone(), bearer, self.transport.clone(), + attach_request_timeout, ) .await } @@ -334,6 +339,7 @@ impl PreparedBackend for OpenShellLaunch { services.sandbox_bearer, ) .with_transport(self.transport) + .with_attach_request_timeout(services.attach_request_timeout) .with_recovery_timeout_secs(services.supervisor_recovery_timeout_secs); if let Some(takeover) = services.takeover { backend = backend.with_takeover(takeover); diff --git a/crates/openshell-supervisor/src/backend_setup/registration.rs b/crates/openshell-supervisor/src/backend_setup/registration.rs index df337cc7f1..978ef65033 100644 --- a/crates/openshell-supervisor/src/backend_setup/registration.rs +++ b/crates/openshell-supervisor/src/backend_setup/registration.rs @@ -99,12 +99,14 @@ impl PreparedBackend for RegisteredLaunch { async fn discover_policy( &self, bearer: SessionBearerTokenSlot, + attach_request_timeout: std::time::Duration, ) -> std::result::Result<(Option, bool), BackendError> { OpenShellRuntimeBackend::discover_registered_policy( self.name.clone(), self.descriptor.clone(), self.payload.clone(), bearer, + attach_request_timeout, ) .await } @@ -119,14 +121,17 @@ impl PreparedBackend for RegisteredLaunch { )); } let payload = self.payload.clone(); - let backend = Arc::new(OpenShellRuntimeBackend::registered( - self.name, - self.descriptor, - self.payload, - services.ca_file_paths, - services.provider_credentials, - services.sandbox_bearer, - )?); + let backend = Arc::new( + OpenShellRuntimeBackend::registered( + self.name, + self.descriptor, + self.payload, + services.ca_file_paths, + services.provider_credentials, + services.sandbox_bearer, + )? + .with_attach_request_timeout(services.attach_request_timeout), + ); Ok(BuiltBackend { backend, payload, diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 42626fbe7c..3bb26c7ef9 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -200,6 +200,7 @@ impl TestSetup { HashMap::from([("TEST_CONFIG".into(), "initial".into())]), ), sandbox_bearer: self.auth.sandbox_bearer_slot().unwrap(), + attach_request_timeout: std::time::Duration::from_mins(5), takeover: None, supervisor_recovery_timeout_secs: None, } @@ -277,6 +278,7 @@ impl PreparedBackend for TestLaunch { async fn discover_policy( &self, bearer: SessionBearerTokenSlot, + _attach_request_timeout: std::time::Duration, ) -> std::result::Result<(Option, bool), BackendError> { self.observed.record("discover"); bearer.authorization_metadata().unwrap(); @@ -853,6 +855,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { auth_bundle: setup.auth.clone(), admitted_isolation_backend: Some(TEST_BACKEND.into()), main_exit_marker: Some(marker.clone()), + attach_request_timeout: Duration::from_mins(5), trace_context: None, provider_workload_api_socket: None, // The binary loads the proxy trust directory from its environment. @@ -1061,6 +1064,7 @@ async fn suspend_signal_exits_without_terminating_boundary() { auth_bundle: setup.auth.clone(), admitted_isolation_backend: Some(TEST_BACKEND.into()), main_exit_marker: Some(marker.clone()), + attach_request_timeout: Duration::from_mins(5), trace_context: None, provider_workload_api_socket: None, // The binary loads the proxy trust directory from its environment. diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index a9dd93b5c2..a28e0cf4bf 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -235,6 +235,7 @@ impl PreparedBackend for HeldPreparedBackend { async fn discover_policy( &self, _bearer: openshell_core::jwt::SessionBearerTokenSlot, + _attach_request_timeout: Duration, ) -> Result<(Option, bool), BackendError> { self.0.discovered.store(true, Ordering::SeqCst); Ok((None, false)) @@ -310,6 +311,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl auth_bundle, admitted_isolation_backend: Some("test-held".into()), main_exit_marker: None, + attach_request_timeout: Duration::from_mins(5), trace_context: None, provider_workload_api_socket: None, proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index b9a5546bf2..7012ecacb1 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -920,6 +920,8 @@ pub struct SandboxInstanceConfig { pub auth_bundle: openshell_core::jwt::SupervisorAuthBundle, pub admitted_isolation_backend: Option, pub main_exit_marker: Option, + /// Initial boundary attachment and image-policy discovery budget. + pub attach_request_timeout: Duration, /// Parent of the startup span. pub trace_context: Option, /// SPIFFE Workload API socket for provider token grants; absent fails closed. @@ -1092,6 +1094,7 @@ pub async fn run_sandbox_instance( auth_bundle, admitted_isolation_backend, main_exit_marker, + attach_request_timeout, trace_context, provider_workload_api_socket, proxy_tls, @@ -1181,7 +1184,7 @@ pub async fn run_sandbox_instance( } let startup_status = startup.in_scope(openshell_otel::ErrorStatusGuard::current); let (image_yaml, invalid_image) = selected_backend - .discover_policy(sandbox_bearer.clone()) + .discover_policy(sandbox_bearer.clone(), attach_request_timeout) .instrument(tracing::info_span!(parent: &startup, "supervisor.boundary.discover_policy")) .await?; let image_discovery = if invalid_image { @@ -1337,6 +1340,7 @@ pub async fn run_sandbox_instance( ca_file_paths: ca_file_paths.clone(), provider_credentials: provider_credentials.clone(), sandbox_bearer, + attach_request_timeout, takeover, supervisor_recovery_timeout_secs, }, @@ -4707,6 +4711,7 @@ mod tests { auth_bundle, admitted_isolation_backend: None, main_exit_marker: None, + attach_request_timeout: Duration::from_mins(5), trace_context: None, provider_workload_api_socket: None, proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles::default(), diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index feb3aea9c8..f0529b8840 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -238,6 +238,17 @@ struct Args { /// Stdin is a driver-owned liveness pipe. EOF means the driver exited. #[arg(long, hide = true)] parent_liveness_stdin: bool, + + /// Attach and policy-discover request timeout, in milliseconds. Slow + /// VM-backed runtimes (CoCo/peer-pods) need longer than the default for + /// the remote boundary to boot before the supervisor gives up. + #[arg( + long, + env = "OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS", + default_value = "300000", + value_parser = clap::value_parser!(u64).range(1000..=86_400_000) + )] + attach_request_timeout_ms: u64, } #[cfg(unix)] @@ -657,6 +668,9 @@ fn main() -> Result<()> { auth_bundle, admitted_isolation_backend: Some(admitted_backend), main_exit_marker: args.main_exit_marker, + attach_request_timeout: std::time::Duration::from_millis( + args.attach_request_timeout_ms, + ), trace_context: openshell_supervisor::trace_context_from_environment(), provider_workload_api_socket: std::env::var_os( openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, @@ -880,6 +894,29 @@ fn supervisor_exit(result: Result) -> Result { #[cfg(test)] mod tests { + #[test] + fn bootstrap_timeout_arguments_reject_invalid_values() { + for value in ["0", "999", "86400001", "invalid"] { + assert!( + Args::try_parse_from([ + "openshell-supervisor", + "--attach-request-timeout-ms", + value + ]) + .is_err() + ); + } + for value in ["1000", "900000", "86400000"] { + assert!( + Args::try_parse_from([ + "openshell-supervisor", + "--attach-request-timeout-ms", + value + ]) + .is_ok() + ); + } + } #[cfg(unix)] #[test] diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 243f181653..358e94b7a2 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -125,6 +125,9 @@ from sandbox runtimes)", instance_root, runtime_listen, removal_grace: openshell_supervisor_control::INSTANCE_REMOVAL_GRACE, + attach_request_timeout: std::time::Duration::from_millis( + args.attach_request_timeout_ms, + ), pool, }, )?; diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index 9e6ae7d241..14086df475 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -175,6 +175,8 @@ pub struct SharedHostConfig { pub runtime_listen: SocketAddr, /// Time a withdrawn sandbox instance may take to stop its workload. pub removal_grace: Duration, + /// Initial boundary attachment and image-policy discovery budget. + pub attach_request_timeout: Duration, /// This process's identity as a member of the supervisor pool. pub pool: PoolIdentity, } @@ -187,6 +189,7 @@ pub struct SharedHost { router: Arc, gateway_endpoint: String, removal_grace: Duration, + attach_request_timeout: Duration, /// Assignments of the latest snapshot that could not be admitted, reported /// as failed instances until the gateway withdraws them. rejected: Mutex>, @@ -305,6 +308,7 @@ impl SharedHost { router, gateway_endpoint: config.gateway_endpoint, removal_grace: config.removal_grace, + attach_request_timeout: config.attach_request_timeout, rejected: Mutex::new(BTreeMap::new()), desired_routes: Mutex::new(None), pool: config.pool, @@ -816,6 +820,7 @@ impl SharedHost { auth_bundle: registration.auth, admitted_isolation_backend: Some(openshell_sandbox_backend::BACKEND_NAME.to_string()), main_exit_marker: None, + attach_request_timeout: self.attach_request_timeout, trace_context: registration.trace_context, // One process cannot hold a per-sandbox workload identity, so // providers that need one fail closed. @@ -1621,6 +1626,7 @@ mod tests { instance_root: root.path().join("instances"), runtime_listen: "127.0.0.1:0".parse().unwrap(), removal_grace: Duration::from_secs(10), + attach_request_timeout: Duration::from_mins(5), pool: PoolIdentity { member_id: "member-1".to_string(), runtime_address: "127.0.0.1:5500".parse().unwrap(), @@ -1657,6 +1663,7 @@ mod tests { instance_root: root.path().join("instances"), runtime_listen: "127.0.0.1:0".parse().unwrap(), removal_grace: Duration::from_secs(10), + attach_request_timeout: Duration::from_mins(5), pool: PoolIdentity { member_id: "member-1".to_string(), runtime_address: "127.0.0.1:5500".parse().unwrap(), @@ -1778,6 +1785,7 @@ mod tests { instance_root: root.path().join("instances"), runtime_listen: "127.0.0.1:0".parse().unwrap(), removal_grace: Duration::from_secs(10), + attach_request_timeout: Duration::from_mins(5), pool: PoolIdentity { member_id: "member-1".to_string(), runtime_address: "127.0.0.1:5500".parse().unwrap(), @@ -1874,6 +1882,7 @@ mod tests { instance_root: root.path().join("instances"), runtime_listen: "127.0.0.1:0".parse().unwrap(), removal_grace: Duration::from_secs(10), + attach_request_timeout: Duration::from_mins(5), pool: PoolIdentity { member_id: "member-1".to_string(), runtime_address: "127.0.0.1:5500".parse().unwrap(), @@ -2094,6 +2103,7 @@ mod tests { instance_root: root.path().join("instances"), runtime_listen: "127.0.0.1:0".parse().unwrap(), removal_grace: Duration::from_secs(10), + attach_request_timeout: Duration::from_mins(5), pool: PoolIdentity { member_id: "member-1".to_string(), runtime_address: "127.0.0.1:5500".parse().unwrap(), @@ -2447,6 +2457,7 @@ mod tests { instance_root: root.path().join("instances"), runtime_listen: "127.0.0.1:0".parse().unwrap(), removal_grace: Duration::from_secs(10), + attach_request_timeout: Duration::from_mins(5), pool: PoolIdentity { member_id: "member-1".to_string(), runtime_address: "127.0.0.1:5500".parse().unwrap(), diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 175836ccb4..90e5076889 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -54,6 +54,28 @@ Debian and RPM user services, place them in `~/.config/openshell/gateway.env`; the service supplies that file to both preflight and gateway startup. +### Slow sandbox startup + +Slow isolation backends may need longer startup windows. These settings default +to `300000` milliseconds (5 minutes); allowed values are `1000` through +`86400000`. Environment variables override TOML values. + +| Setting | Scope | Environment variable | +|---|---|---| +| `provisioning_repair_window_ms` | `[openshell.gateway]`: configuration repair after admission starts | `OPENSHELL_PROVISIONING_REPAIR_WINDOW_MS` | +| `runtime_bootstrap_grace_ms` | `[openshell.drivers.kubernetes]`: sandbox startup | `OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS` | +| `sandbox_attach_request_timeout_ms` | `[openshell.drivers.kubernetes]`: dedicated supervisor startup | `OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS` | + +For a 15-minute Kubernetes startup window, set both driver settings to `900000`. +For shared supervisors, set `OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS=900000` on each +shared host's deployment; the driver setting does not change those deployments. +A directly launched supervisor also accepts `--attach-request-timeout-ms`. + +Restart the affected gateway and supervisors for new settings to take effect. +Existing admission deadlines are preserved. After a sandbox times out, wait for +cleanup, then explicitly start it again. The CLI wait timeout and the separate +image-preparation timeout do not change these windows. + `name` assigns an operator-facing identity to the gateway installation. Set it with `[openshell.gateway].name`, `--name`, or `OPENSHELL_GATEWAY_NAME`. It defaults to `openshell`; the Helm chart defaults it to the chart fullname so all replicas in one installation share a name. Chart fullnames are only unique within their Kubernetes namespace, so set `server.name` explicitly when one collector receives telemetry from multiple namespaces or clusters. This identity is independent of client-side gateway aliases, TLS names, and `gateway_jwt.gateway_id`. ## Package-Managed Locations @@ -282,6 +304,9 @@ ssh_session_ttl_secs = 3600 # Applies to new attempts only; allowed values are 1 through 86400 seconds. image_preparation_timeout_seconds = 1800 +# Admission repair window in milliseconds; existing deadlines are preserved. +provisioning_repair_window_ms = 300000 + # Reject invalid policy generations securely by default. Set # "retain_last_valid" only when availability takes priority. policy_validation_failure_mode = "fail_closed" @@ -1276,6 +1301,9 @@ service_account_name = "openshell-sandbox" default_image = "nvcr.io/nvidia/base/ubuntu:24.04" image_pull_policy = "if_not_present" image_pull_secrets = ["regcred"] +# Startup windows in milliseconds; defaults are 300000. +# runtime_bootstrap_grace_ms = 900000 +# sandbox_attach_request_timeout_ms = 900000 # Defaults to the gateway version; override to pin a specific build. # sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" sandbox_runtime_image_pull_policy = "if_not_present" diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index ec8c9dee6f..558a054577 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -249,7 +249,7 @@ the image/effective policy or provider configuration. The supervisor remains alive while the workload stays unstarted. Inspect `openshell sandbox get` and repair the desired configuration with a complete policy replacement or provider change; do not treat a healthy container as proof that the workload is ready. -If the 300-second provisioning repair window expires, the gateway records +If the provisioning repair window (default 300 seconds) expires, the gateway records `ProvisioningTimedOut` and stops workload and supervisor compute. Inspect `provisioning` in sandbox JSON and TUI NOTES to distinguish cleanup pending from complete. Repairing configuration after expiry does not restart compute: wait @@ -263,6 +263,14 @@ fails discovery within its control-request deadline. Permanent gateway errors and exhausted transient retries terminate startup; inspect those errors as connectivity, authorization, or lifecycle failures. +For slow Kubernetes isolation backends, increase `runtime_bootstrap_grace_ms` +and the dedicated supervisor's `sandbox_attach_request_timeout_ms` together. +Shared supervisor deployments instead need `OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS` +on each shared host. The default is 300000 ms; allowed values are 1000 through +86400000. Restart affected components and explicitly start a timed-out sandbox +after cleanup; changing settings does not reset a persisted admission deadline. +See [gateway configuration](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration#slow-sandbox-startup). + ### Step 4: Check Docker-Backed Gateways The sandbox container's log holds the sandbox runtime's warnings and the From e43b7474e825c0fa99d27b75fe413e64471fe9c9 Mon Sep 17 00:00:00 2001 From: Shiju Date: Fri, 6 Nov 2026 12:54:00 +0530 Subject: [PATCH 2/3] fix(compute): correct configurable startup window diagnostics Signed-off-by: Shiju --- crates/openshell-sandbox-backend/README.md | 5 +++-- crates/openshell-server/src/compute/mod.rs | 22 +++++++++++-------- .../src/compute/provisioning_deadline.rs | 4 ++-- .../how-it-works/policies/manage-policies.mdx | 4 ++-- docs/how-it-works/sandboxes/overview.mdx | 2 +- skills/openshell-cli/SKILL.md | 2 +- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/crates/openshell-sandbox-backend/README.md b/crates/openshell-sandbox-backend/README.md index 8deb207f5d..b64aac2df3 100644 --- a/crates/openshell-sandbox-backend/README.md +++ b/crates/openshell-sandbox-backend/README.md @@ -199,8 +199,9 @@ selects which instance's generation-pinned TLS identity the supervisor then demands. Unknown, removed, and replaced handles are closed before any handshake. Handshakes are bounded globally and per route, and an authenticated connection only fills its own instance's spare queue. A routed -instance waits up to five minutes for its first connection, since it is -registered before its workload Pod is scheduled, and the usual connect window +instance waits up to its configured initial attachment budget (five minutes +by default) for its first connection, since it is registered before its +workload Pod is scheduled, and the usual connect window afterwards. The launch material and status of one hosted instance (`SandboxInstanceRegistration`, `SandboxInstanceStatusReport`) are defined by `openshell-supervisor-control`, which depends on this crate for the runtime diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 87a46ced30..0cf3ea791a 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -19286,7 +19286,10 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" async fn seed_provisioning_attempt(runtime: &ComputeRuntime) -> Sandbox { let mut sandbox = sandbox_record("sb-ttl", "sandbox-ttl", SandboxPhase::Provisioning); let status = sandbox.status.as_mut().unwrap(); - status.provisioning = Some(provisioning_deadline::new_record(0, 300_000)); + status.provisioning = Some(provisioning_deadline::new_record( + 0, + runtime.provisioning_repair_window_ms, + )); status.configuration_activated = Some(false); status.configuration_admission = Some(openshell_core::proto::SandboxConfigurationAdmission { @@ -20987,29 +20990,30 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" #[tokio::test] async fn provisioning_timeout_persists_error_before_cleanup_and_preserves_record() { let driver = ControlledDriver::new(); - let runtime = test_runtime(driver.clone()).await; + let runtime = test_runtime(driver.clone()) + .await + .with_provisioning_repair_window_ms(900_000) + .unwrap(); let sandbox = seed_provisioning_attempt(&runtime).await; let gate = runtime.lifecycle_gates.lock_for("sb-ttl").await; let sandbox_guard = runtime.lock_sandbox_for_lifecycle(&gate).await; assert!( runtime - .claim_provisioning_timeout(&sandbox, 299_999) + .claim_provisioning_timeout(&sandbox, 899_999) .await .unwrap() .is_none() ); let expired = runtime - .claim_provisioning_timeout(&sandbox, 300_000) + .claim_provisioning_timeout(&sandbox, 900_000) .await .unwrap() .unwrap(); drop(sandbox_guard); assert_eq!(expired.phase(), i32::from(SandboxPhase::Error)); - assert!( - ready_condition(&expired) - .unwrap() - .message - .contains("Provider requires an inspected endpoint") + assert_eq!( + ready_condition(&expired).unwrap().message, + "Provisioning repair window expired: Provider requires an inspected endpoint" ); assert_eq!(driver.stop_calls(), 0); runtime diff --git a/crates/openshell-server/src/compute/provisioning_deadline.rs b/crates/openshell-server/src/compute/provisioning_deadline.rs index 2763f86d1a..5f74a7b65f 100644 --- a/crates/openshell-server/src/compute/provisioning_deadline.rs +++ b/crates/openshell-server/src/compute/provisioning_deadline.rs @@ -652,10 +652,10 @@ impl super::ComputeRuntime { let message = if preparation_expired { "Image preparation or initial supervisor startup exceeded its absolute deadline".to_string() } else if diagnostic.is_empty() { - "Provisioning repair window expired after 300 seconds".to_string() + "Provisioning repair window expired".to_string() } else { format!( - "Provisioning repair window expired after 300 seconds: {diagnostic}" + "Provisioning repair window expired: {diagnostic}" ) }; status diff --git a/docs/how-it-works/policies/manage-policies.mdx b/docs/how-it-works/policies/manage-policies.mdx index c20af7b67b..e24cf05f22 100644 --- a/docs/how-it-works/policies/manage-policies.mdx +++ b/docs/how-it-works/policies/manage-policies.mdx @@ -368,11 +368,11 @@ openshell sandbox get my-sandbox --output json The same repair window applies when the gateway refuses an image-policy upload or a policy update that adds baseline filesystem paths with `FAILED_PRECONDITION` or `INVALID_ARGUMENT`. After the gateway acknowledges the rejection report, startup waits two seconds, reads a fresh configuration, and retries. If your repair reaches the gateway before that report, startup immediately reads the repaired configuration. The sandbox log records the gateway's reason; repeated refusals of the same write, status code, and configuration are logged once. Authentication and authorization failures still stop startup. -You have 300 seconds to fix the configuration after the supervisor starts admission. Image preparation has its own deadline and does not consume that repair time. Replace the policy with `openshell policy set`, or fix the provider configuration. Until the workload first starts, you can also change filesystem, Landlock, and process settings. Each effective change to the policy, providers, or settings restarts the 300 seconds; writing an unchanged value does not. +You have the configured repair window (300 seconds by default) to fix the configuration after the supervisor starts admission. Image preparation has its own deadline and does not consume that repair time. Replace the policy with `openshell policy set`, or fix the provider configuration. Until the workload first starts, you can also change filesystem, Landlock, and process settings. Each effective change to the policy, providers, or settings restarts the repair window; writing an unchanged value does not. If the time runs out, the sandbox moves to `Error` with the reason `ProvisioningTimedOut`. Fixing the configuration does not restart it. After you -fix it, start the sandbox again. Admission gets a new 300-second window after preparation: +fix it, start the sandbox again. Admission gets a new repair window after preparation: ```shell openshell sandbox start my-sandbox diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 2ac6a4f69d..6365f9e684 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -1171,7 +1171,7 @@ stopped sandbox repeats configuration admission before launching its workload. Image preparation and initial supervisor startup have an absolute deadline of 1800 seconds by default, independently of the CLI wait timeout. Operators can set `image_preparation_timeout_seconds` in `[openshell.gateway]` to any value from 1 through 86400. Each new create or explicit start stores its deadline. Progress events, configuration edits, and gateway or driver restarts cannot extend it. Changing the gateway setting affects new attempts only. -The first authenticated configuration report from the current supervisor ends preparation and starts a separate 300-second admission repair window. A slow image download therefore does not consume time reserved for fixing policy. During admission, an effective policy, settings, provider, profile, or attachment change resets the window from its stored change time. The first failed configuration load for that change grants another full window. Repeated failures, duplicate reports, and reconnects do not extend it; reaching `Ready` clears it. +The first authenticated configuration report from the current supervisor ends preparation and starts a separate admission repair window (300 seconds by default). A slow image download therefore does not consume time reserved for fixing policy. During admission, an effective policy, settings, provider, profile, or attachment change resets the window from its stored change time. The first failed configuration load for that change grants another full window. Repeated failures, duplicate reports, and reconnects do not extend it; reaching `Ready` clears it. Preparation expiry sets the sandbox to `Error` with reason `ImagePreparationTimedOut`; admission repair expiry uses `ProvisioningTimedOut`. The gateway stops its workload and supervisor compute, retaining the sandbox record, diagnostic, and restartable storage. Cleanup can remain pending if the backend is unavailable; the gateway retries it. Inspect `provisioning` in JSON output for the active `phase`, `deadline`, original `preparation_deadline`, `admission_start_time`, and cleanup timestamps. TUI NOTES identifies preparation timeout and distinguishes pending cleanup from reclaimed compute. diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index 351d7e3b84..98ade9cc78 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -620,7 +620,7 @@ repair the complete policy or provider set through the gateway. The workload has not started on its first activation, so static fields can also be replaced during this initial repair. A previously activated sandbox retains static-field restrictions while restart admission is pending or rejected. -Before the gateway's 300-second repair window expires, successful validation +Before the gateway's configured repair window expires (300 seconds by default), successful validation completes startup in place. Effective stored configuration changes and their first failed load reset that window; repeated failures do not. After `ProvisioningTimedOut`, inspect the retained record and cleanup status, repair From fd6cbd5f15e5efce908010d210ff09ca5c989a2e Mon Sep 17 00:00:00 2001 From: Shiju Date: Sat, 7 Nov 2026 13:39:09 +0530 Subject: [PATCH 3/3] fix(compute): use seconds for operator startup timeouts Signed-off-by: Shiju --- crates/openshell-core/src/config.rs | 8 +-- .../openshell-driver-kubernetes/src/config.rs | 50 +++++++++---------- .../openshell-driver-kubernetes/src/driver.rs | 6 +-- .../openshell-driver-kubernetes/src/main.rs | 38 ++++++++------ .../src/sandbox_runtime.rs | 14 +++--- crates/openshell-gateway/src/lib.rs | 16 +++--- crates/openshell-server/src/cli.rs | 45 ++++++++++------- crates/openshell-server/src/compute/mod.rs | 10 ++-- crates/openshell-server/src/config_file.rs | 4 +- crates/openshell-server/src/grpc/policy.rs | 6 +-- crates/openshell-server/src/grpc/sandbox.rs | 4 +- crates/openshell-server/src/lib.rs | 3 +- crates/openshell-supervisor/src/main.rs | 42 +++++++++------- crates/openshell-supervisor/src/shared.rs | 4 +- docs/how-it-works/gateways/configuration.mdx | 26 +++++----- skills/debug-openshell-cluster/SKILL.md | 10 ++-- 16 files changed, 155 insertions(+), 131 deletions(-) diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index bcf382ce1e..20718b8612 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -328,10 +328,10 @@ pub struct Config { /// sandbox attempts, in seconds. Must be between 1 and 86400, inclusive. pub image_preparation_timeout_seconds: u32, - /// Provisioning repair window, in milliseconds. The first authenticated + /// Provisioning repair window, in seconds. The first authenticated /// supervisor configuration report starts admission repair, which stays - /// armed until Ready. Must be between 1000 and 86400000, inclusive. - pub provisioning_repair_window_ms: i64, + /// armed until Ready. Must be between 1 and 86400, inclusive. + pub provisioning_repair_window_seconds: i64, /// Maximum gRPC requests allowed per rate-limit window. /// @@ -1053,7 +1053,7 @@ impl Config { default_credential_driver: None, ssh_session_ttl_secs: default_ssh_session_ttl_secs(), image_preparation_timeout_seconds: 1800, - provisioning_repair_window_ms: 300_000, + provisioning_repair_window_seconds: 300, grpc_rate_limit_requests: None, grpc_rate_limit_window_secs: None, service_routing: ServiceRoutingConfig::default(), diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 1fa91e34cf..b20f9e32bf 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -347,16 +347,16 @@ pub struct KubernetesComputeConfig { /// Cross-pod sandbox/supervisor settings. pub sandbox_runtime: KubernetesSandboxRuntimeConfig, /// Bound on how long a crash-interrupted, fail-closed sandbox runtime - /// bootstrap may remain stranded before the driver reclaims it. Slow + /// bootstrap may remain stranded before the driver reclaims it, in seconds. Slow /// VM-backed runtimes (CoCo/peer-pods) need longer than the default. - pub runtime_bootstrap_grace_ms: u64, - /// Supervisor attach/policy-discover request timeout, in milliseconds. + pub runtime_bootstrap_grace_seconds: u64, + /// Supervisor attach/policy-discover request timeout, in seconds. /// When set, the driver copies the value into the supervisor pod - /// environment (`OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS`) so slow VM-backed + /// environment (`OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS`) so slow VM-backed /// runtimes have time to boot the remote boundary before the supervisor /// gives up. #[serde(default, skip_serializing_if = "Option::is_none")] - pub sandbox_attach_request_timeout_ms: Option, + pub sandbox_attach_request_timeout_seconds: Option, /// Corporate HTTP forward proxy used by the network supervisor for /// policy-approved TLS CONNECT egress. pub https_proxy: Option, @@ -520,8 +520,8 @@ impl Default for KubernetesComputeConfig { supervisor_image: config::default_supervisor_image(), supervisor_image_pull_policy: None, sandbox_runtime: KubernetesSandboxRuntimeConfig::default(), - runtime_bootstrap_grace_ms: 300_000, - sandbox_attach_request_timeout_ms: None, + runtime_bootstrap_grace_seconds: 300, + sandbox_attach_request_timeout_seconds: None, https_proxy: None, no_proxy: None, proxy_auth_secret_name: None, @@ -566,18 +566,16 @@ impl KubernetesComputeConfig { fn validate_bootstrap_windows(&self) -> Result<(), String> { for (name, value) in [ ( - "runtime_bootstrap_grace_ms", - Some(self.runtime_bootstrap_grace_ms), + "runtime_bootstrap_grace_seconds", + Some(self.runtime_bootstrap_grace_seconds), ), ( - "sandbox_attach_request_timeout_ms", - self.sandbox_attach_request_timeout_ms, + "sandbox_attach_request_timeout_seconds", + self.sandbox_attach_request_timeout_seconds, ), ] { - if value.is_some_and(|value| !(1000..=86_400_000).contains(&value)) { - return Err(format!( - "{name} must be between 1000 and 86400000 milliseconds" - )); + if value.is_some_and(|value| !(1..=86_400).contains(&value)) { + return Err(format!("{name} must be between 1 and 86400 seconds")); } } Ok(()) @@ -1281,33 +1279,33 @@ mod tests { #[test] fn bootstrap_grace_defaults_and_parses_from_toml() { assert_eq!( - KubernetesComputeConfig::default().runtime_bootstrap_grace_ms, - 300_000 + KubernetesComputeConfig::default().runtime_bootstrap_grace_seconds, + 300 ); assert!( KubernetesComputeConfig::default() - .sandbox_attach_request_timeout_ms + .sandbox_attach_request_timeout_seconds .is_none() ); let config: KubernetesComputeConfig = toml::from_str( - "runtime_bootstrap_grace_ms = 900000\nsandbox_attach_request_timeout_ms = 900000\n", + "runtime_bootstrap_grace_seconds = 900\nsandbox_attach_request_timeout_seconds = 900\n", ) .unwrap(); - assert_eq!(config.runtime_bootstrap_grace_ms, 900_000); - assert_eq!(config.sandbox_attach_request_timeout_ms, Some(900_000)); + assert_eq!(config.runtime_bootstrap_grace_seconds, 900); + assert_eq!(config.sandbox_attach_request_timeout_seconds, Some(900)); assert!(config.validate_configuration().is_ok()); let partial: KubernetesComputeConfig = toml::from_str("namespace = 'custom'").unwrap(); - assert_eq!(partial.runtime_bootstrap_grace_ms, 300_000); + assert_eq!(partial.runtime_bootstrap_grace_seconds, 300); for field in [ - "runtime_bootstrap_grace_ms", - "sandbox_attach_request_timeout_ms", + "runtime_bootstrap_grace_seconds", + "sandbox_attach_request_timeout_seconds", ] { - for value in [0, 999, 86_400_001] { + for value in [0, 86_401] { let config: KubernetesComputeConfig = toml::from_str(&format!("{field} = {value}")).unwrap(); assert!(config.validate_configuration().unwrap_err().contains(field)); } - for value in [1000, 86_400_000] { + for value in [1, 86_400] { let config: KubernetesComputeConfig = toml::from_str(&format!("{field} = {value}")).unwrap(); assert!(config.validate_configuration().is_ok()); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 7afe3e4579..78a3739e75 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -3579,7 +3579,7 @@ impl KubernetesComputeDriver { .map(|generation| (generation.name.as_str(), generation.digest.as_str())), owner, &self.supervisor_otlp_environment(), - self.config.sandbox_attach_request_timeout_ms, + self.config.sandbox_attach_request_timeout_seconds, ) .map_err(KubernetesDriverError::Message)?, &self.config.sandbox_runtime.app_listener, @@ -4099,7 +4099,7 @@ impl KubernetesComputeDriver { sandbox_api.api.clone(), cr_name.to_string(), Some(cr_uid.to_string()), - Duration::from_millis(self.config.runtime_bootstrap_grace_ms), + Duration::from_secs(self.config.runtime_bootstrap_grace_seconds), ); Ok(()) } @@ -5541,7 +5541,7 @@ impl KubernetesComputeDriver { if !sandbox_runtime_bootstrap_is_stale( object, SystemTime::now(), - Duration::from_millis(self.config.runtime_bootstrap_grace_ms), + Duration::from_secs(self.config.runtime_bootstrap_grace_seconds), ) { return; } diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index af1669ef03..c2ba057bbd 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -29,14 +29,14 @@ struct Args { default_value = "{}" )] admission_config_json: openshell_core::resource_admission::DriverAdmissionConfig, - /// Runtime bootstrap budget in milliseconds. - #[arg(long, env = "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS", default_value = "300000", - value_parser = clap::value_parser!(u64).range(1000..=86_400_000))] - runtime_bootstrap_grace_ms: u64, + /// Runtime bootstrap budget in seconds. + #[arg(long, env = "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_SECONDS", default_value = "300", + value_parser = clap::value_parser!(u64).range(1..=86_400))] + runtime_bootstrap_grace_seconds: u64, /// Initial attachment budget passed to dedicated supervisor Pods. - #[arg(long, env = "OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS", - value_parser = clap::value_parser!(u64).range(1000..=86_400_000))] - sandbox_attach_request_timeout_ms: Option, + #[arg(long, env = "OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS", + value_parser = clap::value_parser!(u64).range(1..=86_400))] + sandbox_attach_request_timeout_seconds: Option, /// Public compute-driver Unix socket used by an external gateway. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] bind_socket: Option, @@ -294,8 +294,8 @@ async fn main() -> Result<()> { openshell_driver_kubernetes::config::KubernetesAppListenerConfig::default(), boundary_port: args.sandbox_runtime_boundary_port, }, - runtime_bootstrap_grace_ms: args.runtime_bootstrap_grace_ms, - sandbox_attach_request_timeout_ms: args.sandbox_attach_request_timeout_ms, + runtime_bootstrap_grace_seconds: args.runtime_bootstrap_grace_seconds, + sandbox_attach_request_timeout_seconds: args.sandbox_attach_request_timeout_seconds, https_proxy: args.https_proxy, no_proxy: args.no_proxy, proxy_auth_secret_name: args.proxy_auth_secret_name, @@ -377,17 +377,27 @@ async fn main() -> Result<()> { mod tests { #[test] fn bootstrap_timeout_arguments_reject_invalid_values() { + let defaults = Args::try_parse_from(["openshell-driver-kubernetes"]).unwrap(); + assert_eq!(defaults.runtime_bootstrap_grace_seconds, 300); + assert!(defaults.sandbox_attach_request_timeout_seconds.is_none()); for flag in [ - "--runtime-bootstrap-grace-ms", - "--sandbox-attach-request-timeout-ms", + "--runtime-bootstrap-grace-seconds", + "--sandbox-attach-request-timeout-seconds", ] { - for value in ["0", "999", "86400001", "invalid"] { + for value in ["0", "86401", "invalid"] { assert!( Args::try_parse_from(["openshell-driver-kubernetes", flag, value]).is_err() ); } - for value in ["1000", "900000", "86400000"] { - assert!(Args::try_parse_from(["openshell-driver-kubernetes", flag, value]).is_ok()); + for value in ["1", "900", "86400"] { + let args = + Args::try_parse_from(["openshell-driver-kubernetes", flag, value]).unwrap(); + let seconds = value.parse::().unwrap(); + if flag == "--runtime-bootstrap-grace-seconds" { + assert_eq!(args.runtime_bootstrap_grace_seconds, seconds); + } else { + assert_eq!(args.sandbox_attach_request_timeout_seconds, Some(seconds)); + } } } } diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index c596aa6391..7f073a23be 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -353,7 +353,7 @@ pub fn supervisor_pod( network_additional_ca: Option<(&str, &str)>, owner: OwnerReference, tracing_environment: &[(&str, String)], - attach_request_timeout_ms: Option, + attach_request_timeout_seconds: Option, ) -> Result { let labels = control_labels(sandbox_id, gateway_id); let mut environment = vec![ @@ -395,10 +395,10 @@ pub fn supervisor_pod( .iter() .map(|(name, value)| env_var(name, value)), ); - if let Some(timeout_ms) = attach_request_timeout_ms { + if let Some(timeout_seconds) = attach_request_timeout_seconds { environment.push(env_var( - "OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS", - &timeout_ms.to_string(), + "OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS", + &timeout_seconds.to_string(), )); } if let Some(server_name) = gateway_tls_server_name(grpc_endpoint) { @@ -1179,10 +1179,10 @@ mod tests { .map(|variable| (variable.name, variable.value.unwrap_or_default())) .collect::>() }; - assert!(!env(None).contains_key("OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS")); + assert!(!env(None).contains_key("OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS")); assert_eq!( - env(Some(900_000))["OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS"], - "900000" + env(Some(900))["OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS"], + "900" ); } diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 5327940bc7..6395a57598 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -342,31 +342,31 @@ fn kubernetes_config( config.supervisor_image, std::env::var(openshell_core::config::SUPERVISOR_IMAGE_ENV).ok(), ); - if let Some(raw) = std::env::var_os("OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS") { + if let Some(raw) = std::env::var_os("OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_SECONDS") { let raw = raw.to_str().ok_or_else(|| { openshell_core::Error::config( - "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS must be an integer number of milliseconds", + "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_SECONDS must be an integer number of seconds", ) })?; let grace = raw.parse::().map_err(|_| { openshell_core::Error::config( - "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS must be an integer number of milliseconds", + "OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_SECONDS must be an integer number of seconds", ) })?; - config.runtime_bootstrap_grace_ms = grace; + config.runtime_bootstrap_grace_seconds = grace; } - if let Some(raw) = std::env::var_os("OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS") { + if let Some(raw) = std::env::var_os("OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS") { let raw = raw.to_str().ok_or_else(|| { openshell_core::Error::config( - "OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS must be an integer number of milliseconds", + "OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS must be an integer number of seconds", ) })?; let timeout = raw.parse::().map_err(|_| { openshell_core::Error::config( - "OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS must be an integer number of milliseconds", + "OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS must be an integer number of seconds", ) })?; - config.sandbox_attach_request_timeout_ms = Some(timeout); + config.sandbox_attach_request_timeout_seconds = Some(timeout); } if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { config.workspace_default_storage_size = size; diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 702878c68d..e70b7b6b35 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -705,25 +705,25 @@ fn prepare_server_config_with_drivers( config.image_preparation_timeout_seconds = seconds; } - if let Some(raw) = std::env::var_os("OPENSHELL_PROVISIONING_REPAIR_WINDOW_MS") { + if let Some(raw) = std::env::var_os("OPENSHELL_PROVISIONING_REPAIR_WINDOW_SECONDS") { let raw = raw.to_str().ok_or_else(|| { miette::miette!( - "OPENSHELL_PROVISIONING_REPAIR_WINDOW_MS must be an integer number of milliseconds" + "OPENSHELL_PROVISIONING_REPAIR_WINDOW_SECONDS must be an integer number of seconds" ) })?; let window = raw.parse::().map_err(|_| { miette::miette!( - "OPENSHELL_PROVISIONING_REPAIR_WINDOW_MS must be an integer number of milliseconds" + "OPENSHELL_PROVISIONING_REPAIR_WINDOW_SECONDS must be an integer number of seconds" ) })?; - validate_provisioning_repair_window_ms(window)?; - config.provisioning_repair_window_ms = window; + validate_provisioning_repair_window_seconds(window)?; + config.provisioning_repair_window_seconds = window; } else if let Some(window) = file .as_ref() - .and_then(|f| f.openshell.gateway.provisioning_repair_window_ms) + .and_then(|f| f.openshell.gateway.provisioning_repair_window_seconds) { - validate_provisioning_repair_window_ms(window)?; - config.provisioning_repair_window_ms = window; + validate_provisioning_repair_window_seconds(window)?; + config.provisioning_repair_window_seconds = window; } if let Some(issuer) = args.oidc_issuer.clone() { @@ -1641,10 +1641,10 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches: } } -fn validate_provisioning_repair_window_ms(window: i64) -> Result<()> { - if !(1_000..=86_400_000).contains(&window) { +fn validate_provisioning_repair_window_seconds(window: i64) -> Result<()> { + if !(1..=86_400).contains(&window) { return Err(miette::miette!( - "provisioning_repair_window_ms must be between 1000 and 86400000" + "provisioning_repair_window_seconds must be between 1 and 86400" )); } Ok(()) @@ -4973,6 +4973,13 @@ mem_mib = "not-a-number" #[test] fn server_config_prepares_provisioning_repair_window_from_toml() { + assert_eq!( + openshell_core::Config::new(None).provisioning_repair_window_seconds, + 300 + ); + for seconds in [1, 86_400] { + super::validate_provisioning_repair_window_seconds(seconds).unwrap(); + } let state = tempfile::tempdir().unwrap(); let tls = tempfile::tempdir().unwrap(); Environment::new() @@ -4984,7 +4991,7 @@ mem_mib = "not-a-number" let config_path = state.join("gateway.toml"); std::fs::write( &config_path, - "[openshell]\nversion = 2\n[openshell.gateway]\nprovisioning_repair_window_ms = 900000\n", + "[openshell]\nversion = 2\n[openshell.gateway]\nprovisioning_repair_window_seconds = 900\n", ) .unwrap(); let (mut args, matches) = parse_with_args(&[ @@ -4999,7 +5006,7 @@ mem_mib = "not-a-number" ]); let prepared = super::prepare_server_config(&mut args, &matches) .expect("server config is prepared"); - assert_eq!(prepared.config.provisioning_repair_window_ms, 900_000); + assert_eq!(prepared.config.provisioning_repair_window_seconds, 900); }); } @@ -5014,11 +5021,11 @@ mem_mib = "not-a-number" let state = std::path::PathBuf::from(std::env::var_os("XDG_STATE_HOME").unwrap()); let config_path = state.join("gateway.toml"); - for window in [999, 86_400_001] { + for window in [-1, 0, 86_401] { std::fs::write( &config_path, format!( - "[openshell]\nversion = 2\n[openshell.gateway]\nprovisioning_repair_window_ms = {window}\n" + "[openshell]\nversion = 2\n[openshell.gateway]\nprovisioning_repair_window_seconds = {window}\n" ), ) .unwrap(); @@ -5038,7 +5045,7 @@ mem_mib = "not-a-number" assert!( error .to_string() - .contains("provisioning_repair_window_ms must be between 1000 and 86400000") + .contains("provisioning_repair_window_seconds must be between 1 and 86400") ); } }); @@ -5051,14 +5058,14 @@ mem_mib = "not-a-number" Environment::new() .set("XDG_STATE_HOME", state.path().to_str().unwrap()) .set("OPENSHELL_LOCAL_TLS_DIR", tls.path().to_str().unwrap()) - .set("OPENSHELL_PROVISIONING_REPAIR_WINDOW_MS", "600000") + .set("OPENSHELL_PROVISIONING_REPAIR_WINDOW_SECONDS", "600") .run(|| { let state = std::path::PathBuf::from(std::env::var_os("XDG_STATE_HOME").unwrap()); let config_path = state.join("gateway.toml"); std::fs::write( &config_path, - "[openshell]\nversion = 2\n[openshell.gateway]\nprovisioning_repair_window_ms = 900000\n", + "[openshell]\nversion = 2\n[openshell.gateway]\nprovisioning_repair_window_seconds = 900\n", ) .unwrap(); let (mut args, matches) = parse_with_args(&[ @@ -5073,7 +5080,7 @@ mem_mib = "not-a-number" ]); let prepared = super::prepare_server_config(&mut args, &matches) .expect("server config is prepared"); - assert_eq!(prepared.config.provisioning_repair_window_ms, 600_000); + assert_eq!(prepared.config.provisioning_repair_window_seconds, 600); }); } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 0cf3ea791a..d19ad9e4e9 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -1347,14 +1347,14 @@ impl ComputeRuntime { /// Validate the repair window once at startup. Persisted attempts keep /// their original deadline even if the operator changes this value on /// restart. - pub(crate) fn with_provisioning_repair_window_ms( + pub(crate) fn with_provisioning_repair_window_seconds( mut self, window: i64, ) -> Result { - if !(1_000..=86_400_000).contains(&window) { - return Err("provisioning_repair_window_ms must be between 1000 and 86400000".into()); + if !(1..=86_400).contains(&window) { + return Err("provisioning_repair_window_seconds must be between 1 and 86400".into()); } - self.provisioning_repair_window_ms = window; + self.provisioning_repair_window_ms = window * 1000; Ok(self) } @@ -20992,7 +20992,7 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" let driver = ControlledDriver::new(); let runtime = test_runtime(driver.clone()) .await - .with_provisioning_repair_window_ms(900_000) + .with_provisioning_repair_window_seconds(900) .unwrap(); let sandbox = seed_provisioning_attempt(&runtime).await; let gate = runtime.lifecycle_gates.lock_for("sb-ttl").await; diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index ce3f2838fd..fcc76fbd7c 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -149,9 +149,9 @@ pub struct GatewayFileSection { /// Absolute preparation budget for new attempts; existing deadlines persist. #[serde(default)] pub image_preparation_timeout_seconds: Option, - /// Admission repair window in milliseconds, between 1000 and 86400000. + /// Admission repair window in seconds, between 1 and 86400. #[serde(default)] - pub provisioning_repair_window_ms: Option, + pub provisioning_repair_window_seconds: Option, /// Explicit activation only after the compatible deployment rollout ends. #[serde(default)] pub enable_sandbox_updates: Option, diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 604dd38112..eb6ea70dd6 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -5103,7 +5103,7 @@ pub(super) async fn handle_report_sandbox_configuration( &state.store, &mut sandbox, current_time_ms(), - state.config.provisioning_repair_window_ms, + state.config.provisioning_repair_window_seconds * 1000, ) .await .map_err(Status::internal)?; @@ -5200,14 +5200,14 @@ pub(super) async fn handle_report_sandbox_configuration( crate::compute::provisioning_deadline::record_admission_start( record, now_ms, - state.config.provisioning_repair_window_ms, + state.config.provisioning_repair_window_seconds * 1000, ) .map_err(Status::internal)?; if reported == ConfigurationAdmissionState::Rejected { crate::compute::provisioning_deadline::record_rejection( record, now_ms, - state.config.provisioning_repair_window_ms, + state.config.provisioning_repair_window_seconds * 1000, ) .map_err(Status::internal)?; } diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 80d857612e..2ecb85f7f8 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -741,14 +741,14 @@ async fn handle_create_sandbox_inner( crate::compute::provisioning_deadline::new_preparation_record( now_ms, state.config.image_preparation_timeout_seconds, - state.config.provisioning_repair_window_ms, + state.config.provisioning_repair_window_seconds * 1000, ), ); crate::compute::provisioning_deadline::refresh_configuration( &state.store, &mut sandbox, now_ms, - state.config.provisioning_repair_window_ms, + state.config.provisioning_repair_window_seconds * 1000, ) .await .map_err(Status::internal)?; diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index e5fa3c1986..9d9dd0c676 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -2285,7 +2285,8 @@ async fn build_compute_runtime( runtime.with_image_preparation_timeout(config.image_preparation_timeout_seconds) }) .and_then(|runtime| { - runtime.with_provisioning_repair_window_ms(config.provisioning_repair_window_ms) + runtime + .with_provisioning_repair_window_seconds(config.provisioning_repair_window_seconds) }) .map_err(Error::config)?; Ok(runtime.with_telemetry_compute_driver(telemetry_compute_driver)) diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index f0529b8840..b98d1abc89 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -239,16 +239,16 @@ struct Args { #[arg(long, hide = true)] parent_liveness_stdin: bool, - /// Attach and policy-discover request timeout, in milliseconds. Slow + /// Attach and policy-discover request timeout, in seconds. Slow /// VM-backed runtimes (CoCo/peer-pods) need longer than the default for /// the remote boundary to boot before the supervisor gives up. #[arg( long, - env = "OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS", - default_value = "300000", - value_parser = clap::value_parser!(u64).range(1000..=86_400_000) + env = "OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS", + default_value = "300", + value_parser = clap::value_parser!(u64).range(1..=86_400) )] - attach_request_timeout_ms: u64, + attach_request_timeout_seconds: u64, } #[cfg(unix)] @@ -668,8 +668,8 @@ fn main() -> Result<()> { auth_bundle, admitted_isolation_backend: Some(admitted_backend), main_exit_marker: args.main_exit_marker, - attach_request_timeout: std::time::Duration::from_millis( - args.attach_request_timeout_ms, + attach_request_timeout: std::time::Duration::from_secs( + args.attach_request_timeout_seconds, ), trace_context: openshell_supervisor::trace_context_from_environment(), provider_workload_api_socket: std::env::var_os( @@ -896,24 +896,32 @@ fn supervisor_exit(result: Result) -> Result { mod tests { #[test] fn bootstrap_timeout_arguments_reject_invalid_values() { - for value in ["0", "999", "86400001", "invalid"] { + assert_eq!( + Args::try_parse_from(["openshell-supervisor"]) + .unwrap() + .attach_request_timeout_seconds, + 300 + ); + for value in ["0", "86401", "invalid"] { assert!( Args::try_parse_from([ "openshell-supervisor", - "--attach-request-timeout-ms", + "--attach-request-timeout-seconds", value ]) .is_err() ); } - for value in ["1000", "900000", "86400000"] { - assert!( - Args::try_parse_from([ - "openshell-supervisor", - "--attach-request-timeout-ms", - value - ]) - .is_ok() + for value in ["1", "900", "86400"] { + let args = Args::try_parse_from([ + "openshell-supervisor", + "--attach-request-timeout-seconds", + value, + ]) + .unwrap(); + assert_eq!( + args.attach_request_timeout_seconds, + value.parse::().unwrap() ); } } diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index 358e94b7a2..aa35b527a8 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -125,8 +125,8 @@ from sandbox runtimes)", instance_root, runtime_listen, removal_grace: openshell_supervisor_control::INSTANCE_REMOVAL_GRACE, - attach_request_timeout: std::time::Duration::from_millis( - args.attach_request_timeout_ms, + attach_request_timeout: std::time::Duration::from_secs( + args.attach_request_timeout_seconds, ), pool, }, diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 90e5076889..ab04e93c8a 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -57,19 +57,19 @@ preflight and gateway startup. ### Slow sandbox startup Slow isolation backends may need longer startup windows. These settings default -to `300000` milliseconds (5 minutes); allowed values are `1000` through -`86400000`. Environment variables override TOML values. +to `300` seconds (5 minutes); allowed values are `1` through +`86400`. Environment variables override TOML values. | Setting | Scope | Environment variable | |---|---|---| -| `provisioning_repair_window_ms` | `[openshell.gateway]`: configuration repair after admission starts | `OPENSHELL_PROVISIONING_REPAIR_WINDOW_MS` | -| `runtime_bootstrap_grace_ms` | `[openshell.drivers.kubernetes]`: sandbox startup | `OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_MS` | -| `sandbox_attach_request_timeout_ms` | `[openshell.drivers.kubernetes]`: dedicated supervisor startup | `OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS` | +| `provisioning_repair_window_seconds` | `[openshell.gateway]`: configuration repair after admission starts | `OPENSHELL_PROVISIONING_REPAIR_WINDOW_SECONDS` | +| `runtime_bootstrap_grace_seconds` | `[openshell.drivers.kubernetes]`: sandbox startup | `OPENSHELL_RUNTIME_BOOTSTRAP_GRACE_SECONDS` | +| `sandbox_attach_request_timeout_seconds` | `[openshell.drivers.kubernetes]`: dedicated supervisor startup | `OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS` | -For a 15-minute Kubernetes startup window, set both driver settings to `900000`. -For shared supervisors, set `OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS=900000` on each +For a 15-minute Kubernetes startup window, set both driver settings to `900`. +For shared supervisors, set `OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS=900` on each shared host's deployment; the driver setting does not change those deployments. -A directly launched supervisor also accepts `--attach-request-timeout-ms`. +A directly launched supervisor also accepts `--attach-request-timeout-seconds`. Restart the affected gateway and supervisors for new settings to take effect. Existing admission deadlines are preserved. After a sandbox times out, wait for @@ -304,8 +304,8 @@ ssh_session_ttl_secs = 3600 # Applies to new attempts only; allowed values are 1 through 86400 seconds. image_preparation_timeout_seconds = 1800 -# Admission repair window in milliseconds; existing deadlines are preserved. -provisioning_repair_window_ms = 300000 +# Admission repair window in seconds; existing deadlines are preserved. +provisioning_repair_window_seconds = 300 # Reject invalid policy generations securely by default. Set # "retain_last_valid" only when availability takes priority. @@ -1301,9 +1301,9 @@ service_account_name = "openshell-sandbox" default_image = "nvcr.io/nvidia/base/ubuntu:24.04" image_pull_policy = "if_not_present" image_pull_secrets = ["regcred"] -# Startup windows in milliseconds; defaults are 300000. -# runtime_bootstrap_grace_ms = 900000 -# sandbox_attach_request_timeout_ms = 900000 +# Startup windows in seconds; defaults are 300. +# runtime_bootstrap_grace_seconds = 900 +# sandbox_attach_request_timeout_seconds = 900 # Defaults to the gateway version; override to pin a specific build. # sandbox_runtime_image = "ghcr.io/nvidia/openshell/sandbox:" sandbox_runtime_image_pull_policy = "if_not_present" diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 558a054577..e6cae310b7 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -263,11 +263,11 @@ fails discovery within its control-request deadline. Permanent gateway errors and exhausted transient retries terminate startup; inspect those errors as connectivity, authorization, or lifecycle failures. -For slow Kubernetes isolation backends, increase `runtime_bootstrap_grace_ms` -and the dedicated supervisor's `sandbox_attach_request_timeout_ms` together. -Shared supervisor deployments instead need `OPENSHELL_ATTACH_REQUEST_TIMEOUT_MS` -on each shared host. The default is 300000 ms; allowed values are 1000 through -86400000. Restart affected components and explicitly start a timed-out sandbox +For slow Kubernetes isolation backends, increase `runtime_bootstrap_grace_seconds` +and the dedicated supervisor's `sandbox_attach_request_timeout_seconds` together. +Shared supervisor deployments instead need `OPENSHELL_ATTACH_REQUEST_TIMEOUT_SECONDS` +on each shared host. The default is 300 seconds; allowed values are 1 through +86400. Restart affected components and explicitly start a timed-out sandbox after cleanup; changing settings does not reset a persisted admission deadline. See [gateway configuration](https://docs.nvidia.com/openshell/latest/how-it-works/gateways/configuration#slow-sandbox-startup).