diff --git a/crates/openshell-cli/src/commands/gateway.rs b/crates/openshell-cli/src/commands/gateway.rs index 705976f7af..eafea8b148 100644 --- a/crates/openshell-cli/src/commands/gateway.rs +++ b/crates/openshell-cli/src/commands/gateway.rs @@ -46,6 +46,7 @@ struct ComputeDriverInfoView { struct ComputeDriverCapabilitiesView { driver_name: String, driver_version: String, + resources: Option, } #[derive(Debug, Clone)] @@ -417,6 +418,7 @@ pub async fn gateway_info( capabilities: ComputeDriverCapabilitiesView { driver_name: capabilities.driver_name, driver_version: capabilities.driver_version, + resources: capabilities.resource_capabilities, }, } }) @@ -514,9 +516,125 @@ fn print_compute_driver_info(drivers: &[ComputeDriverInfoView]) { "Driver version:".dimmed(), driver.capabilities.driver_version ); + if let Some(resources) = &driver.capabilities.resources { + for (name, supported, limit, request) in [ + ( + "CPU", + resources.cpu.as_ref().map(|c| c.limit_supported), + resources + .cpu + .as_ref() + .and_then(|c| c.limit_enforcement.as_ref()), + resources + .cpu + .as_ref() + .and_then(|c| c.request_enforcement.as_ref()), + ), + ( + "Memory", + resources.memory.as_ref().map(|c| c.limit_supported), + resources + .memory + .as_ref() + .and_then(|c| c.limit_enforcement.as_ref()), + resources + .memory + .as_ref() + .and_then(|c| c.request_enforcement.as_ref()), + ), + ] { + let guarantee = if supported == Some(false) { + "unsupported".into() + } else { + enforcement_label(limit) + }; + println!( + " {name} limits: {guarantee}; requests: {}", + enforcement_label(request) + ); + } + if let Some(sampled) = &resources.sampled_budget { + println!( + " Sampled budgets: CPU time={}, resident bytes={}, scope={}; terminate on excess={}", + sampled.cpu_time_supported, + sampled.resident_bytes_supported, + scope_name(sampled.scope), + sampled.terminate_on_excess + ); + let interval = sampled.sampling_interval.map_or_else( + || "unreported".into(), + |d| format!("{}.{:09} s", d.seconds, d.nanos), + ); + println!( + " Sampling interval: {interval} (plus collection time; overshoot unbounded)" + ); + let resolution = sampled.cpu_time_resolution.map_or_else( + || "unreported".into(), + |d| format!("{}.{:09} s", d.seconds, d.nanos), + ); + println!( + " CPU-time resolution: {resolution}; per-process CPU limit: {}", + sampled.per_process_cpu_limit + ); + } + } else { + println!(" Resource guarantees: unreported"); + } + } +} + +fn scope_name(scope: i32) -> &'static str { + use openshell_core::proto::sandbox::v1::ResourceEnforcementScope as Scope; + match Scope::try_from(scope) { + Ok(Scope::Process) => "process", + Ok(Scope::Container) => "container", + Ok(Scope::SandboxAccount) => "sandbox account", + Ok(Scope::VirtualMachine) => "virtual machine", + _ => "unreported", } } +fn enforcement_label( + enforcement: Option<&openshell_core::proto::sandbox::v1::ResourceEnforcement>, +) -> String { + use openshell_core::proto::sandbox::v1::ResourceEnforcementKind as Kind; + let Some(enforcement) = enforcement else { + return "unreported".into(); + }; + let kind = match Kind::try_from(enforcement.kind) { + Ok(Kind::KernelLimit) => "kernel limit", + Ok(Kind::SchedulerRequest) => "scheduler request", + Ok(Kind::Unsupported) => "unsupported", + _ => "unreported", + }; + format!("{kind} ({})", scope_name(enforcement.scope)) +} + +fn resource_capabilities_json( + resources: &openshell_core::proto::ResourceCapabilities, +) -> serde_json::Value { + fn enforcement( + e: Option<&openshell_core::proto::sandbox::v1::ResourceEnforcement>, + ) -> serde_json::Value { + e.map_or( + serde_json::Value::Null, + |e| serde_json::json!({"kind": e.kind, "scope": e.scope}), + ) + } + fn duration(d: Option) -> serde_json::Value { + d.map_or( + serde_json::Value::Null, + |d| serde_json::json!({"seconds": d.seconds.to_string(), "nanos": d.nanos}), + ) + } + serde_json::json!({ + "cpu": resources.cpu.as_ref().map(|c| serde_json::json!({"limit_supported": c.limit_supported, "limit_enforcement": enforcement(c.limit_enforcement.as_ref()), "request_enforcement": enforcement(c.request_enforcement.as_ref())})), + "memory": resources.memory.as_ref().map(|c| serde_json::json!({"limit_supported": c.limit_supported, "limit_enforcement": enforcement(c.limit_enforcement.as_ref()), "request_enforcement": enforcement(c.request_enforcement.as_ref())})), + "gpu": resources.gpu.as_ref().map(|c| serde_json::json!({"default_selection_supported": c.default_selection_supported, "count_selection_supported": c.count_selection_supported})), + "sampled_budget": resources.sampled_budget.as_ref().map(|c| serde_json::json!({"cpu_time_supported": c.cpu_time_supported, "resident_bytes_supported": c.resident_bytes_supported, "scope": c.scope, "sampling_interval": duration(c.sampling_interval), "terminate_on_excess": c.terminate_on_excess, "cpu_time_resolution": duration(c.cpu_time_resolution), "per_process_cpu_limit": c.per_process_cpu_limit})), + }) +} + fn gateway_info_to_json(view: &GatewayInfoView) -> serde_json::Value { serde_json::json!({ "gateway": &view.gateway, @@ -532,6 +650,7 @@ fn gateway_info_to_json(view: &GatewayInfoView) -> serde_json::Value { "capabilities": { "driver_name": &driver.capabilities.driver_name, "driver_version": &driver.capabilities.driver_version, + "resource_capabilities": driver.capabilities.resources.as_ref().map(resource_capabilities_json), }, })) .collect::>(), @@ -1905,6 +2024,31 @@ mod tests { capabilities: ComputeDriverCapabilitiesView { driver_name: "podman".to_string(), driver_version: "0.0.75".to_string(), + resources: Some(openshell_core::proto::ResourceCapabilities { + cpu: Some(openshell_core::proto::CpuResourceCapabilities { + limit_supported: true, + limit_enforcement: Some( + openshell_core::proto::sandbox::v1::ResourceEnforcement { + kind: 2, + scope: 2, + }, + ), + ..Default::default() + }), + sampled_budget: Some( + openshell_core::proto::sandbox::v1::SampledResourceBudgetCapabilities { + resident_bytes_supported: true, + scope: 3, + sampling_interval: Some(prost_types::Duration { + seconds: 0, + nanos: 100_000_000, + }), + terminate_on_excess: true, + ..Default::default() + }, + ), + ..Default::default() + }), }, }], extensions: vec![ExtensionInfoView { @@ -1928,6 +2072,16 @@ mod tests { json["compute_drivers"][0]["capabilities"]["driver_name"], "podman" ); + let resources = &json["compute_drivers"][0]["capabilities"]["resource_capabilities"]; + assert_eq!( + resources["cpu"]["limit_enforcement"], + serde_json::json!({"kind": 2, "scope": 2}) + ); + assert_eq!( + resources["sampled_budget"]["sampling_interval"]["nanos"], + 100_000_000 + ); + assert_eq!(resources["sampled_budget"]["terminate_on_excess"], true); assert_eq!(json["extensions"][0]["protocol_version"], "1.0"); assert_eq!( json["compute_drivers"][0]["capabilities"]["driver_version"], diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 7411629483..ab2febd4f9 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -1997,6 +1997,16 @@ enum SandboxTemplateCommands { #[arg(long)] memory: Option, + /// Sampled cumulative CPU-time budget (for example: 30s). Terminates on + /// observed excess; this is not a CPU-rate limit. + #[arg(long, value_name = "DURATION")] + cpu_time_budget: Option, + + /// Sampled account-wide resident-memory budget in bytes. Terminates on + /// observed excess; this is not a hard memory limit. + #[arg(long, value_name = "BYTES", value_parser = clap::value_parser!(u64).range(1..))] + resident_memory_budget_bytes: Option, + /// Request GPU resources for sandboxes created from this template. /// /// Omit COUNT for the driver's default GPU selection, or pass COUNT @@ -3764,6 +3774,8 @@ async fn run_async() -> Result<()> { image, cpu, memory, + cpu_time_budget, + resident_memory_budget_bytes, gpu, driver_config_json, ready_within, @@ -3786,6 +3798,10 @@ async fn run_async() -> Result<()> { image.as_deref(), cpu.as_deref(), memory.as_deref(), + run::sampled_resource_budget( + cpu_time_budget.as_deref(), + resident_memory_budget_bytes, + )?, gpu_requirements, driver_config_json.as_deref(), ready_within.as_deref(), @@ -6657,6 +6673,10 @@ mod tests { "2", "--memory", "4Gi", + "--cpu-time-budget", + "30s", + "--resident-memory-budget-bytes", + "1073741824", "--gpu", "1", "--driver-config-json", @@ -6685,6 +6705,8 @@ mod tests { image, cpu, memory, + cpu_time_budget, + resident_memory_budget_bytes, gpu, driver_config_json, ready_within, @@ -6697,6 +6719,8 @@ mod tests { })), .. }) => { + assert_eq!(cpu_time_budget.as_deref(), Some("30s")); + assert_eq!(resident_memory_budget_bytes, Some(1_073_741_824)); assert_eq!(name, "gpu-kata"); assert_eq!(image.as_deref(), Some("registry.example.com/agent:latest")); assert_eq!(cpu.as_deref(), Some("2")); diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index eb224685ed..259d72e1c6 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -651,7 +651,10 @@ pub async fn sandbox_create( None }; - let resource_requirements = gpu_requirements.map(|gpu| ResourceRequirements { gpu: Some(gpu) }); + let resource_requirements = gpu_requirements.map(|gpu| ResourceRequirements { + gpu: Some(gpu), + ..Default::default() + }); let main_terminal = tty_override .unwrap_or_else(|| std::io::stdin().is_terminal() && std::io::stdout().is_terminal()); @@ -3380,6 +3383,7 @@ pub async fn sandbox_template_create( image: Option<&str>, cpu: Option<&str>, memory: Option<&str>, + sampled_budget: Option, gpu_requirements: Option, driver_config_json: Option<&str>, ready_within: Option<&str>, @@ -3392,7 +3396,11 @@ pub async fn sandbox_template_create( tls: &TlsOptions, suppress_credential_warnings: bool, ) -> Result<()> { - let resources = if cpu.is_some() || memory.is_some() || gpu_requirements.is_some() { + let resources = if cpu.is_some() + || memory.is_some() + || gpu_requirements.is_some() + || sampled_budget.is_some() + { Some(SandboxResources { cpu: cpu .map(validate_cpu_quantity) @@ -3403,6 +3411,7 @@ pub async fn sandbox_template_create( .transpose()? .unwrap_or_default(), gpu: gpu_requirements, + sampled_budget, }) } else { None @@ -3413,6 +3422,19 @@ pub async fn sandbox_template_create( let desired_service_level = build_template_service_level(ready_within, max_burst)?; let mut client = grpc_client(server, tls).await?; + if sampled_budget.is_some() + && !client + .health(openshell_core::proto::HealthRequest {}) + .await + .into_diagnostic()? + .into_inner() + .supports_sampled_resource_budget + { + return Err(miette!( + "gateway does not acknowledge sampled resource budgets; upgrade it before creating this template" + )); + } + let profile_catalog = fetch_provider_profile_catalog(&mut client, workspace) .await .unwrap_or_default(); @@ -3463,6 +3485,25 @@ pub async fn sandbox_template_create( Ok(()) } +pub fn sampled_resource_budget( + cpu_time: Option<&str>, + resident_bytes: Option, +) -> Result> { + if cpu_time.is_none() && resident_bytes.is_none() { + return Ok(None); + } + let budget = openshell_core::proto::sandbox::v1::SampledResourceBudget { + cpu_time: cpu_time + .map(parse_duration_to_ms) + .transpose()? + .map(duration_ms_to_proto), + resident_bytes, + }; + openshell_core::resource_contract::validate_sampled_budget(&budget) + .map_err(|error| miette!("{error}"))?; + Ok(Some(budget)) +} + fn build_template_service_level( ready_within: Option<&str>, max_burst: Option, @@ -3691,6 +3732,12 @@ fn sandbox_template_to_json(template: &SandboxWorkloadTemplate) -> serde_json::V .map_or_else(|| serde_json::json!("default"), serde_json::Value::from); resources_json.insert("gpu".to_string(), value); } + if let Some(budget) = &resources.sampled_budget { + resources_json.insert("sampled_budget".into(), serde_json::json!({ + "cpu_time": budget.cpu_time.map(|d| serde_json::json!({"seconds": d.seconds.to_string(), "nanos": d.nanos})), + "resident_bytes": budget.resident_bytes.map(|n| n.to_string()), + })); + } if !resources_json.is_empty() { obj.insert( "resources".to_string(), @@ -3777,6 +3824,18 @@ fn print_sandbox_template_detail(template: &SandboxWorkloadTemplate) { workload.environment.len() ); if let Some(resources) = &workload.resources { + if let Some(budget) = &resources.sampled_budget { + if let Some(cpu) = budget.cpu_time.as_ref() { + println!( + " Sampled CPU-time budget: {}.{:09} s", + cpu.seconds, cpu.nanos + ); + } + if let Some(resident) = budget.resident_bytes { + println!(" Sampled resident-memory budget: {resident} bytes"); + } + println!(" Budgets terminate on observed excess; sampling can overshoot."); + } println!( " {} {}", "CPU:".dimmed(), @@ -8262,6 +8321,7 @@ mod tests { fn provisioning_timeout_message_includes_condition_and_gpu_hint() { let resource_requirements = ResourceRequirements { gpu: Some(GpuResourceRequirements { count: None }), + ..Default::default() }; let message = provisioning_timeout_message( 120, @@ -8283,7 +8343,10 @@ mod tests { #[test] fn provisioning_timeout_message_omits_gpu_hint_without_gpu_requirements() { - let resource_requirements = ResourceRequirements { gpu: None }; + let resource_requirements = ResourceRequirements { + gpu: None, + ..Default::default() + }; let message = provisioning_timeout_message(120, Some(&resource_requirements), None); assert_eq!(message, "sandbox provisioning timed out after 120s"); diff --git a/crates/openshell-cli/tests/ensure_providers_integration.rs b/crates/openshell-cli/tests/ensure_providers_integration.rs index be3e4f2e49..3228a99bc8 100644 --- a/crates/openshell-cli/tests/ensure_providers_integration.rs +++ b/crates/openshell-cli/tests/ensure_providers_integration.rs @@ -152,6 +152,7 @@ impl OpenShell for TestOpenShell { Ok(Response::new(HealthResponse { status: ServiceStatus::Healthy.into(), version: "test".to_string(), + supports_sampled_resource_budget: false, })) } diff --git a/crates/openshell-cli/tests/mtls_integration.rs b/crates/openshell-cli/tests/mtls_integration.rs index 1017712035..355487858c 100644 --- a/crates/openshell-cli/tests/mtls_integration.rs +++ b/crates/openshell-cli/tests/mtls_integration.rs @@ -104,6 +104,7 @@ impl OpenShell for TestOpenShell { Ok(Response::new(HealthResponse { status: ServiceStatus::Healthy.into(), version: "test".to_string(), + supports_sampled_resource_budget: false, })) } diff --git a/crates/openshell-cli/tests/provider_commands_integration.rs b/crates/openshell-cli/tests/provider_commands_integration.rs index 89247ba718..5d4ec1c4a3 100644 --- a/crates/openshell-cli/tests/provider_commands_integration.rs +++ b/crates/openshell-cli/tests/provider_commands_integration.rs @@ -307,6 +307,7 @@ impl OpenShell for TestOpenShell { Ok(Response::new(HealthResponse { status: ServiceStatus::Healthy.into(), version: "test".to_string(), + supports_sampled_resource_budget: false, })) } diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index 587910fbaf..d80428358e 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -60,6 +60,7 @@ fn selected_workspace( #[derive(Clone, Default)] struct SandboxState { + sampled_budget_ack: Arc, /// Make `ListProviderProfiles` fail while every other RPC keeps working, /// so a catalog lookup failure can be told apart from an empty catalog. fail_list_provider_profiles: Arc, @@ -165,6 +166,7 @@ impl OpenShell for TestOpenShell { Ok(Response::new(HealthResponse { status: ServiceStatus::Healthy.into(), version: "test".to_string(), + supports_sampled_resource_budget: self.state.sampled_budget_ack.load(Ordering::SeqCst), })) } @@ -2330,26 +2332,36 @@ async fn sandbox_template_create_sends_non_default_workspace_in_scope_and_metada .run_async(async { let tls = test_tls(&server); - run::sandbox_template_create( - &server.endpoint, - "gpu-kata", - Some("registry.example.com/agent:latest"), - Some("2"), - Some("4Gi"), - Some(GpuResourceRequirements { count: Some(1) }), - Some(r#"{"kubernetes":{"pod":{"node_selector":{"pool":"gpu"}}}}"#), - Some("5m"), - Some(3), - HashMap::from([("team".to_string(), "runtime".to_string())]), - HashMap::from([("owner".to_string(), "platform".to_string())]), - HashMap::from([("FEATURE_FLAG".to_string(), "on".to_string())]), - "table", - "team-a", - &tls, - true, - ) - .await - .expect("template create should succeed"); + let create = || { + run::sandbox_template_create( + &server.endpoint, + "gpu-kata", + Some("registry.example.com/agent:latest"), + Some("2"), + Some("4Gi"), + run::sampled_resource_budget(Some("30s"), Some(1_073_741_824)).unwrap(), + Some(GpuResourceRequirements { count: Some(1) }), + Some(r#"{"kubernetes":{"pod":{"node_selector":{"pool":"gpu"}}}}"#), + Some("5m"), + Some(3), + HashMap::from([("team".to_string(), "runtime".to_string())]), + HashMap::from([("owner".to_string(), "platform".to_string())]), + HashMap::from([("FEATURE_FLAG".to_string(), "on".to_string())]), + "table", + "team-a", + &tls, + true, + ) + }; + let error = create().await.unwrap_err(); + assert!(format!("{error:?}").contains("does not acknowledge")); + assert!(template_create_requests(&server).await.is_empty()); + server + .openshell + .state + .sampled_budget_ack + .store(true, Ordering::SeqCst); + create().await.expect("template create should succeed"); let requests = template_create_requests(&server).await; let request = requests @@ -2377,6 +2389,12 @@ async fn sandbox_template_create_sends_non_default_workspace_in_scope_and_metada .resources .as_ref() .expect("resources should be sent"); + let budget = resources + .sampled_budget + .as_ref() + .expect("sampled budget must survive the CLI request"); + assert_eq!(budget.cpu_time.as_ref().unwrap().seconds, 30); + assert_eq!(budget.resident_bytes, Some(1_073_741_824)); assert_eq!(resources.cpu, "2"); assert_eq!(resources.memory, "4Gi"); assert_eq!(resources.gpu.as_ref().and_then(|gpu| gpu.count), Some(1)); @@ -2475,6 +2493,7 @@ async fn sandbox_template_create_allows_omitted_image() { None, None, None, + None, HashMap::new(), HashMap::new(), HashMap::new(), diff --git a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs index 497f3ce778..82312dd58e 100644 --- a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs +++ b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs @@ -119,6 +119,7 @@ impl OpenShell for TestOpenShell { Ok(Response::new(HealthResponse { status: ServiceStatus::Healthy.into(), version: "test".to_string(), + supports_sampled_resource_budget: false, })) } diff --git a/crates/openshell-core/src/extension_protocol.rs b/crates/openshell-core/src/extension_protocol.rs index d6d93418d8..f75a9a3bf8 100644 --- a/crates/openshell-core/src/extension_protocol.rs +++ b/crates/openshell-core/src/extension_protocol.rs @@ -19,6 +19,9 @@ pub const PROTOCOL_MINOR: u32 = 0; /// configured signer before accepting sandbox creation. pub const COMPUTE_LAUNCH_AUTHENTICATION: &str = "openshell.compute.launch-authentication"; +/// Driver describes resource guarantees and admits explicit sampled budgets. +pub const COMPUTE_RESOURCE_CONTRACT: &str = "openshell.compute.resource-contract"; + /// Driver stages operator-selected external isolation routes for the supervisor. pub const COMPUTE_ISOLATION_BACKEND: &str = "openshell.compute.isolation-backend"; @@ -133,6 +136,7 @@ pub fn gateway_metadata(family: ExtensionFamily) -> PeerMetadata { supported_capabilities.push(COMPUTE_LAUNCH_AUTHENTICATION.to_string()); supported_capabilities.push(COMPUTE_ISOLATION_BACKEND.to_string()); supported_capabilities.push(COMPUTE_UPDATE_SANDBOX_CONFIGURATION.to_string()); + supported_capabilities.push(COMPUTE_RESOURCE_CONTRACT.to_string()); } PeerMetadata { protocol_version: Some(ProtocolVersion { diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 12d6cd26ff..6fafd7ae10 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -56,6 +56,7 @@ pub mod proto_struct; pub mod provider_credentials; pub mod replica_routing; pub mod resource_admission; +pub mod resource_contract; pub mod rpc_error; pub mod sandbox_env; pub mod sandbox_generation; diff --git a/crates/openshell-core/src/resource_contract.rs b/crates/openshell-core/src/resource_contract.rs new file mode 100644 index 0000000000..d996de8d13 --- /dev/null +++ b/crates/openshell-core/src/resource_contract.rs @@ -0,0 +1,328 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +//! Admission of resource guarantees, separate from operator host-resource policy. + +use crate::proto::compute::v1::{DriverSandbox, ResourceCapabilities}; +use crate::proto::sandbox::v1::{ + ResourceEnforcement, ResourceEnforcementKind as Kind, ResourceEnforcementScope as Scope, + SampledResourceBudget, +}; + +/// Canonical positive protobuf duration, in nanoseconds without narrowing. +pub fn positive_duration_nanos(value: &prost_types::Duration) -> Result { + if !(0..=315_576_000_000).contains(&value.seconds) || !(0..1_000_000_000).contains(&value.nanos) + { + return Err("resource duration must be positive and canonical"); + } + let nanos = u128::from(value.seconds.unsigned_abs()) * 1_000_000_000 + + u128::from(value.nanos.unsigned_abs()); + if nanos == 0 { + return Err("resource duration must be positive"); + } + Ok(nanos) +} + +pub fn validate_sampled_budget(budget: &SampledResourceBudget) -> Result<(), &'static str> { + if budget.cpu_time.is_none() && budget.resident_bytes.is_none() { + return Err("sampled resource budget must select CPU time or resident bytes"); + } + if let Some(cpu) = &budget.cpu_time { + positive_duration_nanos(cpu)?; + } + if budget.resident_bytes == Some(0) { + return Err("sampled resident bytes must be positive"); + } + Ok(()) +} + +fn require_enforcement( + enforcement: Option<&ResourceEnforcement>, + kind: Kind, +) -> Result<(), &'static str> { + let Some(enforcement) = enforcement else { + return Err("requested resource guarantee is unreported by the compute driver"); + }; + if Kind::try_from(enforcement.kind).ok() != Some(kind) + || !matches!( + Scope::try_from(enforcement.scope), + Ok(Scope::Container | Scope::SandboxAccount | Scope::VirtualMachine) + ) + { + return Err("requested resource guarantee is unsupported at sandbox scope"); + } + Ok(()) +} + +/// Validate both independent CPU/memory axes before any provisioning. +/// +/// Legacy drivers retain their existing hard-limit behavior, but cannot accept the new +/// sampled request by silently ignoring an unknown protobuf field. +pub fn validate_request( + sandbox: &DriverSandbox, + capabilities: Option<&ResourceCapabilities>, + negotiated: bool, +) -> Result<(), &'static str> { + let spec = sandbox.spec.as_ref(); + let resources = spec + .and_then(|s| s.template.as_ref()) + .and_then(|t| t.resources.as_ref()); + if negotiated && let Some(resources) = resources { + let cpu = capabilities.and_then(|c| c.cpu.as_ref()); + let memory = capabilities.and_then(|c| c.memory.as_ref()); + if !resources.cpu_limit.is_empty() { + if !cpu.is_some_and(|c| c.limit_supported) { + return Err("compute driver does not support a hard CPU-rate limit"); + } + require_enforcement( + cpu.and_then(|c| c.limit_enforcement.as_ref()), + Kind::KernelLimit, + )?; + } + if !resources.memory_limit.is_empty() { + if !memory.is_some_and(|c| c.limit_supported) { + return Err("compute driver does not support a hard memory limit"); + } + require_enforcement( + memory.and_then(|c| c.limit_enforcement.as_ref()), + Kind::KernelLimit, + )?; + } + if !resources.cpu_request.is_empty() { + require_enforcement( + cpu.and_then(|c| c.request_enforcement.as_ref()), + Kind::SchedulerRequest, + )?; + } + if !resources.memory_request.is_empty() { + require_enforcement( + memory.and_then(|c| c.request_enforcement.as_ref()), + Kind::SchedulerRequest, + )?; + } + } + if let Some(budget) = spec + .and_then(|s| s.resource_requirements.as_ref()) + .and_then(|r| r.sampled_budget.as_ref()) + { + validate_sampled_budget(budget)?; + if !negotiated { + return Err("sampled budgets require openshell.compute.resource-contract"); + } + let Some(cap) = capabilities.and_then(|c| c.sampled_budget.as_ref()) else { + return Err("compute driver does not support sampled resource budgets"); + }; + if !cap.terminate_on_excess + || Scope::try_from(cap.scope).ok() != Some(Scope::SandboxAccount) + { + return Err("sampled budgets require terminating account-wide observation"); + } + positive_duration_nanos( + cap.sampling_interval + .as_ref() + .ok_or("sampling interval is unreported")?, + )?; + if let Some(cpu) = &budget.cpu_time { + if !cap.cpu_time_supported { + return Err("sampled CPU-time budget is unsupported"); + } + let resolution = positive_duration_nanos( + cap.cpu_time_resolution + .as_ref() + .ok_or("CPU-time resolution is unreported")?, + )?; + if positive_duration_nanos(cpu)? % resolution != 0 { + return Err( + "sampled CPU-time budget cannot be represented at the driver resolution", + ); + } + } + if budget.resident_bytes.is_some() && !cap.resident_bytes_supported { + return Err("sampled resident-memory budget is unsupported"); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::proto::compute::v1::{ + CpuResourceCapabilities, DriverResourceRequirements, DriverSandboxSpec, + DriverSandboxTemplate, MemoryResourceCapabilities, ResourceRequirements, + }; + use crate::proto::sandbox::v1::SampledResourceBudgetCapabilities; + + #[test] + fn resource_guarantees_validate_both_axes_and_never_infer_support() { + let mut sandbox = DriverSandbox { + spec: Some(DriverSandboxSpec { + template: Some(DriverSandboxTemplate { + resources: Some(DriverResourceRequirements::default()), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + for (cpu, memory) in [("1", ""), ("", "512Mi")] { + let resources = sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .resources + .as_mut() + .unwrap(); + resources.cpu_limit = cpu.into(); + resources.memory_limit = memory.into(); + for (kind, scope, accepted) in [ + (Kind::KernelLimit as i32, Scope::Container as i32, true), + (Kind::KernelLimit as i32, Scope::Process as i32, false), + ( + Kind::SchedulerRequest as i32, + Scope::Container as i32, + false, + ), + (99, Scope::Container as i32, false), + (Kind::KernelLimit as i32, 99, false), + ] { + let enforcement = Some(ResourceEnforcement { kind, scope }); + let capabilities = ResourceCapabilities { + cpu: Some(CpuResourceCapabilities { + limit_supported: true, + limit_enforcement: enforcement, + ..Default::default() + }), + memory: Some(MemoryResourceCapabilities { + limit_supported: true, + limit_enforcement: enforcement, + ..Default::default() + }), + ..Default::default() + }; + assert_eq!( + validate_request(&sandbox, Some(&capabilities), true).is_ok(), + accepted + ); + } + assert!(validate_request(&sandbox, None, true).is_err()); + assert!(validate_request(&sandbox, None, false).is_ok()); + } + for (cpu, memory) in [("1", ""), ("", "512Mi")] { + let resources = sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .resources + .as_mut() + .unwrap(); + *resources = DriverResourceRequirements { + cpu_request: cpu.into(), + memory_request: memory.into(), + ..Default::default() + }; + for (kind, scope, accepted) in [ + (Kind::SchedulerRequest as i32, Scope::Container as i32, true), + (Kind::KernelLimit as i32, Scope::Container as i32, false), + (Kind::SchedulerRequest as i32, Scope::Process as i32, false), + (99, Scope::Container as i32, false), + ] { + let enforcement = Some(ResourceEnforcement { kind, scope }); + let capabilities = ResourceCapabilities { + cpu: Some(CpuResourceCapabilities { + request_enforcement: enforcement, + ..Default::default() + }), + memory: Some(MemoryResourceCapabilities { + request_enforcement: enforcement, + ..Default::default() + }), + ..Default::default() + }; + assert_eq!( + validate_request(&sandbox, Some(&capabilities), true).is_ok(), + accepted + ); + } + assert!( + validate_request(&sandbox, Some(&ResourceCapabilities::default()), true).is_err() + ); + } + sandbox.spec.as_mut().unwrap().template = None; + let capabilities = ResourceCapabilities { + sampled_budget: Some(SampledResourceBudgetCapabilities { + cpu_time_supported: true, + resident_bytes_supported: true, + scope: Scope::SandboxAccount as i32, + sampling_interval: Some(prost_types::Duration { + seconds: 0, + nanos: 100_000_000, + }), + cpu_time_resolution: Some(prost_types::Duration { + seconds: 0, + nanos: 1_000_000, + }), + terminate_on_excess: true, + per_process_cpu_limit: true, + }), + ..Default::default() + }; + for (seconds, nanos, resident, accepted) in [ + (0, 1_000_000, None, true), + (1, 0, Some(4096), true), + (0, 1, None, false), + (0, 0, None, false), + (-1, 0, None, false), + (1, -1, None, false), + (1, 1_000_000_000, None, false), + (315_576_000_001, 0, None, false), + (1, 0, Some(0), false), + ] { + sandbox.spec.as_mut().unwrap().resource_requirements = Some(ResourceRequirements { + sampled_budget: Some(SampledResourceBudget { + cpu_time: Some(prost_types::Duration { seconds, nanos }), + resident_bytes: resident, + }), + ..Default::default() + }); + assert_eq!( + validate_request(&sandbox, Some(&capabilities), true).is_ok(), + accepted + ); + assert!(validate_request(&sandbox, Some(&capabilities), false).is_err()); + } + sandbox.spec.as_mut().unwrap().resource_requirements = Some(ResourceRequirements { + sampled_budget: Some(SampledResourceBudget { + cpu_time: Some(prost_types::Duration { + seconds: 1, + nanos: 0, + }), + resident_bytes: Some(4096), + }), + ..Default::default() + }); + for change in 0..8 { + let mut changed = capabilities; + let sampled = changed.sampled_budget.as_mut().unwrap(); + match change { + 0 => sampled.terminate_on_excess = false, + 1 => sampled.scope = Scope::Process as i32, + 2 => sampled.cpu_time_supported = false, + 3 => sampled.resident_bytes_supported = false, + 4 => sampled.sampling_interval = None, + 5 => sampled.cpu_time_resolution = None, + 6 => sampled.sampling_interval.as_mut().unwrap().nanos = -1, + _ => changed.sampled_budget = None, + } + assert!( + validate_request(&sandbox, Some(&changed), true).is_err(), + "changed capability {change}" + ); + } + } +} diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index aa1d7de706..eecc8f6e68 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -1063,6 +1063,36 @@ impl DockerComputeDriver { Ok(driver) } + fn resource_capabilities(gpu_supported: bool) -> ResourceCapabilities { + ResourceCapabilities { + cpu: Some(CpuResourceCapabilities { + limit_supported: true, + limit_enforcement: Some(openshell_core::proto::sandbox::v1::ResourceEnforcement { + kind: openshell_core::proto::sandbox::v1::ResourceEnforcementKind::KernelLimit + as i32, + scope: openshell_core::proto::sandbox::v1::ResourceEnforcementScope::Container + as i32, + }), + ..Default::default() + }), + memory: Some(MemoryResourceCapabilities { + limit_supported: true, + limit_enforcement: Some(openshell_core::proto::sandbox::v1::ResourceEnforcement { + kind: openshell_core::proto::sandbox::v1::ResourceEnforcementKind::KernelLimit + as i32, + scope: openshell_core::proto::sandbox::v1::ResourceEnforcementScope::Container + as i32, + }), + ..Default::default() + }), + gpu: Some(GpuResourceCapabilities { + default_selection_supported: gpu_supported, + count_selection_supported: gpu_supported, + }), + sampled_budget: None, + } + } + fn capabilities(&self) -> GetCapabilitiesResponse { GetCapabilitiesResponse { resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { @@ -1076,25 +1106,14 @@ impl DockerComputeDriver { gateway_manages_lifecycle: true, supports_sandbox_authentication: false, driver_reports_runtime_readiness: false, - resource_capabilities: Some(ResourceCapabilities { - cpu: Some(CpuResourceCapabilities { - limit_supported: true, - }), - memory: Some(MemoryResourceCapabilities { - limit_supported: true, - }), - gpu: Some(GpuResourceCapabilities { - default_selection_supported: self.config.gpu.cdi_supported, - count_selection_supported: self.config.gpu.cdi_supported, - }), - }), + resource_capabilities: Some(Self::resource_capabilities(self.config.gpu.cdi_supported)), rootfs_tar_staging_dir: String::new(), rootfs_tar_max_bytes: 0, extension: Some(openshell_core::extension_protocol::extension_metadata( openshell_core::extension_protocol::ExtensionFamily::Compute, "openshell/docker", openshell_core::VERSION, - [], + [openshell_core::extension_protocol::COMPUTE_RESOURCE_CONTRACT.to_string()], )), } } @@ -1116,6 +1135,12 @@ impl DockerComputeDriver { config.allow_driver_config, sandbox, )?; + openshell_core::resource_contract::validate_request( + sandbox, + Some(&Self::resource_capabilities(config.gpu.cdi_supported)), + true, + ) + .map_err(Status::failed_precondition)?; let spec = sandbox .spec .as_ref() @@ -6471,7 +6496,6 @@ fn docker_pids_limit(value: Option) -> Result, Ok(value.map(std::num::NonZeroI64::get)) } -#[allow(clippy::cast_possible_truncation)] fn parse_cpu_limit(value: &str) -> Result, Status> { let value = value.trim(); if value.is_empty() { @@ -6488,7 +6512,9 @@ fn parse_cpu_limit(value: &str) -> Result, Status> { "docker cpu_limit must be greater than zero", )); } - return Ok(Some(millicores.saturating_mul(1_000_000))); + return millicores.checked_mul(1_000_000).map(Some).ok_or_else(|| { + Status::failed_precondition("docker CPU limit exceeds the supported range") + }); } let cores = value.parse::().map_err(|_| { @@ -6502,10 +6528,9 @@ fn parse_cpu_limit(value: &str) -> Result, Status> { )); } - Ok(Some((cores * 1_000_000_000.0).round() as i64)) + checked_docker_limit(cores * 1_000_000_000.0, "CPU").map(Some) } -#[allow(clippy::cast_possible_truncation)] fn parse_memory_limit(value: &str) -> Result, Status> { let value = value.trim(); if value.is_empty() { @@ -6548,7 +6573,20 @@ fn parse_memory_limit(value: &str) -> Result, Status> { } }; - Ok(Some((amount * multiplier).round() as i64)) + checked_docker_limit(amount * multiplier, "memory").map(Some) +} + +// Docker uses zero as an unlimited control. Positive authored quantities must +// remain positive after conversion and fit the signed wire field exactly. +#[allow(clippy::cast_possible_truncation)] +fn checked_docker_limit(value: f64, resource: &str) -> Result { + let value = value.floor(); + if !value.is_finite() || value < 1.0 || value >= 2_f64.powi(63) { + return Err(Status::failed_precondition(format!( + "docker {resource} limit is outside the supported positive range" + ))); + } + Ok(value as i64) } fn sandbox_from_container_summary(summary: &ContainerSummary) -> Option { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 70fc34f256..44c865e97b 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -326,6 +326,7 @@ fn list_string_driver_config(field: &str, values: &[&str]) -> prost_types::Struc fn gpu_resources(count: Option) -> ResourceRequirements { ResourceRequirements { gpu: Some(GpuResourceRequirements { count }), + ..Default::default() } } @@ -1416,6 +1417,9 @@ async fn tracing_in_process_stream_leaves_status_unset_when_dropped() { fn parse_cpu_limit_supports_cores_and_millicores() { assert_eq!(parse_cpu_limit("250m").unwrap(), Some(250_000_000)); assert_eq!(parse_cpu_limit("2").unwrap(), Some(2_000_000_000)); + for value in ["0.0000000001", "1e100", "9223372036854775807m"] { + assert!(parse_cpu_limit(value).is_err(), "{value}"); + } assert!(parse_cpu_limit("0").is_err()); } @@ -1423,6 +1427,9 @@ fn parse_cpu_limit_supports_cores_and_millicores() { fn parse_memory_limit_supports_binary_quantities() { assert_eq!(parse_memory_limit("512Mi").unwrap(), Some(536_870_912)); assert_eq!(parse_memory_limit("1G").unwrap(), Some(1_000_000_000)); + for value in ["0.1", "999999999999999999999Ei"] { + assert!(parse_memory_limit(value).is_err(), "{value}"); + } assert!(parse_memory_limit("12XB").is_err()); } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index c08a567eed..f8fc81aaab 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1296,6 +1296,38 @@ impl KubernetesComputeDriver { Ok(driver) } + fn resource_capabilities() -> ResourceCapabilities { + ResourceCapabilities { + cpu: Some(CpuResourceCapabilities { + limit_supported: true, + request_enforcement: Some(openshell_core::proto::sandbox::v1::ResourceEnforcement { + kind: openshell_core::proto::sandbox::v1::ResourceEnforcementKind::SchedulerRequest as i32, + scope: openshell_core::proto::sandbox::v1::ResourceEnforcementScope::Container as i32, + }), + limit_enforcement: Some(openshell_core::proto::sandbox::v1::ResourceEnforcement { + kind: openshell_core::proto::sandbox::v1::ResourceEnforcementKind::KernelLimit as i32, + scope: openshell_core::proto::sandbox::v1::ResourceEnforcementScope::Container as i32, + }), +}), + memory: Some(MemoryResourceCapabilities { + limit_supported: true, + request_enforcement: Some(openshell_core::proto::sandbox::v1::ResourceEnforcement { + kind: openshell_core::proto::sandbox::v1::ResourceEnforcementKind::SchedulerRequest as i32, + scope: openshell_core::proto::sandbox::v1::ResourceEnforcementScope::Container as i32, + }), + limit_enforcement: Some(openshell_core::proto::sandbox::v1::ResourceEnforcement { + kind: openshell_core::proto::sandbox::v1::ResourceEnforcementKind::KernelLimit as i32, + scope: openshell_core::proto::sandbox::v1::ResourceEnforcementScope::Container as i32, + }), +}), + gpu: Some(GpuResourceCapabilities { + default_selection_supported: true, + count_selection_supported: true, + }), + sampled_budget: None, +} + } + pub fn capabilities(&self) -> Result { Ok(GetCapabilitiesResponse { resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { @@ -1309,18 +1341,7 @@ impl KubernetesComputeDriver { gateway_manages_lifecycle: false, supports_sandbox_authentication: true, driver_reports_runtime_readiness: false, - resource_capabilities: Some(ResourceCapabilities { - cpu: Some(CpuResourceCapabilities { - limit_supported: true, - }), - memory: Some(MemoryResourceCapabilities { - limit_supported: true, - }), - gpu: Some(GpuResourceCapabilities { - default_selection_supported: true, - count_selection_supported: true, - }), - }), + resource_capabilities: Some(Self::resource_capabilities()), rootfs_tar_staging_dir: String::new(), rootfs_tar_max_bytes: 0, extension: Some(openshell_core::extension_protocol::extension_metadata( @@ -1330,6 +1351,7 @@ impl KubernetesComputeDriver { [ openshell_core::extension_protocol::COMPUTE_UPDATE_SANDBOX_CONFIGURATION .to_string(), + openshell_core::extension_protocol::COMPUTE_RESOURCE_CONTRACT.to_string(), ], )), }) @@ -2105,6 +2127,12 @@ impl KubernetesComputeDriver { self.config.allow_driver_config, sandbox, )?; + openshell_core::resource_contract::validate_request( + sandbox, + Some(&Self::resource_capabilities()), + true, + ) + .map_err(tonic::Status::failed_precondition)?; self.config .resource_admission .validate() @@ -2554,6 +2582,12 @@ impl KubernetesComputeDriver { ) )] pub async fn create_sandbox(&self, sandbox: &Sandbox) -> Result { + openshell_core::resource_contract::validate_request( + sandbox, + Some(&Self::resource_capabilities()), + true, + ) + .map_err(|error| admission_error(tonic::Status::failed_precondition(error)))?; let span_status = openshell_otel::ErrorStatusGuard::current(); let _guard = self .lifecycle_gates @@ -11846,6 +11880,7 @@ mod tests { spec: Some(SandboxSpec { resource_requirements: Some(ResourceRequirements { gpu: Some(GpuResourceRequirements { count: Some(0) }), + ..Default::default() }), ..SandboxSpec::default() }), diff --git a/crates/openshell-driver-mxc/src/driver.rs b/crates/openshell-driver-mxc/src/driver.rs index 2772f1a69c..d2197b513d 100644 --- a/crates/openshell-driver-mxc/src/driver.rs +++ b/crates/openshell-driver-mxc/src/driver.rs @@ -480,12 +480,14 @@ impl MxcComputeBackend { openshell_core::extension_protocol::ExtensionFamily::Compute, "openshell/mxc", openshell_core::VERSION, - [], + [openshell_core::extension_protocol::COMPUTE_RESOURCE_CONTRACT.to_string()], )), } } fn validate_sandbox_fields(sandbox: &DriverSandbox) -> Result<(), tonic::Status> { + openshell_core::resource_contract::validate_request(sandbox, None, true) + .map_err(tonic::Status::failed_precondition)?; if let Some(spec) = &sandbox.spec { if effective_driver_gpu_count(driver_gpu_requirements( spec.resource_requirements.as_ref(), diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 9943d7a0b6..2cd72dfb8a 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -863,8 +863,38 @@ fn build_labels(sandbox: &DriverSandbox) -> BTreeMap { labels } -/// Parse resource limits from the sandbox template, falling back to defaults. -fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) -> ResourceLimits { +/// Reject requests that must not silently become default limits. +pub fn validate_resource_limits(sandbox: &DriverSandbox) -> Result<(), String> { + let resources = sandbox + .spec + .as_ref() + .and_then(|s| s.template.as_ref()) + .and_then(|t| t.resources.as_ref()); + if let Some(resources) = resources { + if !resources.cpu_request.is_empty() || !resources.memory_request.is_empty() { + return Err("podman does not support resource reservations".into()); + } + if !resources.cpu_limit.is_empty() + && parse_cpu_to_microseconds(&resources.cpu_limit).is_none() + { + return Err("invalid podman CPU limit".into()); + } + if !resources.memory_limit.is_empty() + && parse_memory_to_bytes(&resources.memory_limit) + .is_none_or(|n| n == 0 || i64::try_from(n).is_err()) + { + return Err("invalid podman memory limit".into()); + } + } + Ok(()) +} + +/// Defaults apply only when a limit was not requested. +fn build_resource_limits( + sandbox: &DriverSandbox, + config: &PodmanComputeConfig, +) -> Result { + validate_resource_limits(sandbox).map_err(ComputeDriverError::Precondition)?; let resources = sandbox .spec .as_ref() @@ -881,14 +911,14 @@ fn build_resource_limits(sandbox: &DriverSandbox, config: &PodmanComputeConfig) .and_then(|r| parse_memory_to_bytes(&r.memory_limit)) .unwrap_or(DEFAULT_MEMORY_LIMIT); - ResourceLimits { + Ok(ResourceLimits { cpu: CpuLimits { quota: cpu_micros, period: DEFAULT_CPU_PERIOD, }, memory: MemoryLimits { limit: mem_bytes }, pids: podman_pids_limit(config.sandbox_pids_limit).map(|limit| PidsLimits { limit }), - } + }) } fn podman_pids_limit(value: Option) -> Option { @@ -1285,7 +1315,7 @@ fn build_base_spec( .map(|path| path.display().to_string()) .unwrap_or_default(), ); - let resource_limits = build_resource_limits(sandbox, config); + let resource_limits = build_resource_limits(sandbox, config)?; let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts) .map_err(ComputeDriverError::InvalidArgument)?; driver_mounts::validate_workspace_mount_targets( @@ -1884,15 +1914,19 @@ fn parse_cpu_to_microseconds(quantity: &str) -> Option { millis.checked_mul(100)? } else { let cores: f64 = quantity.parse().ok()?; - if cores <= 0.0 || cores.is_nan() || cores.is_infinite() { + if cores <= 0.0 || !cores.is_finite() || cores * 100_000.0 >= 2_f64.powi(63) { return None; } #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] let val = (cores * 100_000.0) as u64; val }; - // A quota of 0 microseconds is invalid — treat as no limit. - if micros == 0 { None } else { Some(micros) } + // OCI CPU quotas are positive signed 64-bit microseconds. + if micros == 0 || i64::try_from(micros).is_err() { + None + } else { + Some(micros) + } } /// Parse a Kubernetes-style memory quantity to bytes. @@ -2072,6 +2106,59 @@ mod tests { let _ = std::fs::remove_file(artifact); } + #[test] + fn resource_limits_reject_invalid_values_and_reservations_without_defaults() { + use openshell_core::proto::compute::v1::{ + DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, + }; + for resources in [ + DriverResourceRequirements { + cpu_limit: "garbage".into(), + ..Default::default() + }, + DriverResourceRequirements { + cpu_limit: "1e100".into(), + ..Default::default() + }, + DriverResourceRequirements { + cpu_limit: "92233720368547759m".into(), + ..Default::default() + }, + DriverResourceRequirements { + memory_limit: "9223372036854775808".into(), + ..Default::default() + }, + DriverResourceRequirements { + memory_limit: "garbage".into(), + ..Default::default() + }, + DriverResourceRequirements { + memory_limit: "0".into(), + ..Default::default() + }, + DriverResourceRequirements { + cpu_request: "1".into(), + ..Default::default() + }, + DriverResourceRequirements { + memory_request: "1Gi".into(), + ..Default::default() + }, + ] { + let sandbox = DriverSandbox { + spec: Some(DriverSandboxSpec { + template: Some(DriverSandboxTemplate { + resources: Some(resources), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + assert!(build_resource_limits(&sandbox, &test_config()).is_err()); + } + } + #[test] fn isolated_pair_keeps_privileges_network_and_secrets_out_of_workload() { let sandbox = DriverSandbox { @@ -2341,6 +2428,7 @@ mod tests { fn gpu_resources(count: Option) -> ResourceRequirements { ResourceRequirements { gpu: Some(GpuResourceRequirements { count }), + ..Default::default() } } @@ -2396,7 +2484,7 @@ mod tests { ..Default::default() }); let config = test_config(); - let limits = build_resource_limits(&sandbox, &config); + let limits = build_resource_limits(&sandbox, &config).unwrap(); assert_eq!(limits.cpu.quota, 50_000); assert_eq!(limits.memory.limit, 2 * 1024 * 1024 * 1024); @@ -2414,7 +2502,7 @@ mod tests { let sandbox = test_sandbox("test-id", "test-name"); let mut config = test_config(); config.sandbox_pids_limit = None; - let limits = build_resource_limits(&sandbox, &config); + let limits = build_resource_limits(&sandbox, &config).unwrap(); assert!(limits.pids.is_none()); } diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 6bea5b7f16..9ae04c22b0 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -713,6 +713,36 @@ impl PodmanComputeDriver { } /// Report driver capabilities. + fn resource_capabilities() -> ResourceCapabilities { + ResourceCapabilities { + cpu: Some(CpuResourceCapabilities { + limit_supported: true, + limit_enforcement: Some(openshell_core::proto::sandbox::v1::ResourceEnforcement { + kind: openshell_core::proto::sandbox::v1::ResourceEnforcementKind::KernelLimit + as i32, + scope: openshell_core::proto::sandbox::v1::ResourceEnforcementScope::Container + as i32, + }), + ..Default::default() + }), + memory: Some(MemoryResourceCapabilities { + limit_supported: true, + limit_enforcement: Some(openshell_core::proto::sandbox::v1::ResourceEnforcement { + kind: openshell_core::proto::sandbox::v1::ResourceEnforcementKind::KernelLimit + as i32, + scope: openshell_core::proto::sandbox::v1::ResourceEnforcementScope::Container + as i32, + }), + ..Default::default() + }), + gpu: Some(GpuResourceCapabilities { + default_selection_supported: true, + count_selection_supported: true, + }), + sampled_budget: None, + } + } + pub fn capabilities(&self) -> Result { Ok(GetCapabilitiesResponse { resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { @@ -726,25 +756,14 @@ impl PodmanComputeDriver { gateway_manages_lifecycle: true, supports_sandbox_authentication: false, driver_reports_runtime_readiness: false, - resource_capabilities: Some(ResourceCapabilities { - cpu: Some(CpuResourceCapabilities { - limit_supported: true, - }), - memory: Some(MemoryResourceCapabilities { - limit_supported: true, - }), - gpu: Some(GpuResourceCapabilities { - default_selection_supported: true, - count_selection_supported: true, - }), - }), + resource_capabilities: Some(Self::resource_capabilities()), rootfs_tar_staging_dir: String::new(), rootfs_tar_max_bytes: 0, extension: Some(openshell_core::extension_protocol::extension_metadata( openshell_core::extension_protocol::ExtensionFamily::Compute, "openshell/podman", openshell_core::VERSION, - [], + [openshell_core::extension_protocol::COMPUTE_RESOURCE_CONTRACT.to_string()], )), }) } @@ -772,6 +791,13 @@ impl PodmanComputeDriver { sandbox, ) .map_err(|error| ComputeDriverError::Precondition(error.message().into()))?; + openshell_core::resource_contract::validate_request( + sandbox, + Some(&Self::resource_capabilities()), + true, + ) + .map_err(|error| ComputeDriverError::Precondition(error.into()))?; + container::validate_resource_limits(sandbox).map_err(ComputeDriverError::Precondition)?; let gpu_requirements = sandbox .spec .as_ref() @@ -2774,6 +2800,7 @@ mod tests { fn gpu_resources(count: Option) -> ResourceRequirements { ResourceRequirements { gpu: Some(GpuResourceRequirements { count }), + ..Default::default() } } diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index f7946c34c2..03ab10acf5 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1534,6 +1534,24 @@ impl VmDriver { } #[must_use] + fn resource_capabilities(gpu_supported: bool) -> ResourceCapabilities { + ResourceCapabilities { + cpu: Some(CpuResourceCapabilities { + limit_supported: false, + ..Default::default() + }), + memory: Some(MemoryResourceCapabilities { + limit_supported: false, + ..Default::default() + }), + gpu: Some(GpuResourceCapabilities { + default_selection_supported: gpu_supported, + count_selection_supported: gpu_supported, + }), + sampled_budget: None, + } + } + pub fn capabilities(&self) -> GetCapabilitiesResponse { let mut extension = openshell_core::extension_protocol::extension_metadata( openshell_core::extension_protocol::ExtensionFamily::Compute, @@ -1544,6 +1562,9 @@ impl VmDriver { extension .required_capabilities .push(openshell_core::extension_protocol::COMPUTE_LAUNCH_AUTHENTICATION.to_string()); + extension + .supported_capabilities + .push(openshell_core::extension_protocol::COMPUTE_RESOURCE_CONTRACT.to_string()); GetCapabilitiesResponse { resource_admission_policy: openshell_core::resource_admission::DriverAdmissionConfig { allow_driver_config: self.config.allow_driver_config, @@ -1556,18 +1577,7 @@ impl VmDriver { gateway_manages_lifecycle: true, supports_sandbox_authentication: false, driver_reports_runtime_readiness: false, - resource_capabilities: Some(ResourceCapabilities { - cpu: Some(CpuResourceCapabilities { - limit_supported: false, - }), - memory: Some(MemoryResourceCapabilities { - limit_supported: false, - }), - gpu: Some(GpuResourceCapabilities { - default_selection_supported: self.config.gpu_enabled, - count_selection_supported: self.config.gpu_enabled, - }), - }), + resource_capabilities: Some(Self::resource_capabilities(self.config.gpu_enabled)), rootfs_tar_staging_dir: self .config .rootfs_tar_staging_dir() @@ -5541,6 +5551,8 @@ fn decode_launch_authentication( #[allow(clippy::result_large_err)] fn validate_vm_sandbox(sandbox: &Sandbox, gpu_enabled: bool) -> Result<(), Status> { + openshell_core::resource_contract::validate_request(sandbox, None, true) + .map_err(Status::failed_precondition)?; validate_sandbox_id(&sandbox.id)?; let spec = sandbox @@ -9575,6 +9587,7 @@ mod tests { fn gpu_resources(count: Option) -> ResourceRequirements { ResourceRequirements { gpu: Some(GpuResourceRequirements { count }), + ..Default::default() } } @@ -9946,7 +9959,7 @@ mod tests { } #[test] - fn validate_vm_sandbox_accepts_template_resources_as_noop() { + fn validate_vm_sandbox_rejects_unenforced_template_resources() { use openshell_core::proto::compute::v1::DriverResourceRequirements; let sandbox = Sandbox { @@ -9964,8 +9977,10 @@ mod tests { }), ..Default::default() }; - validate_vm_sandbox(&sandbox, false) - .expect("template.resources should be accepted and ignored"); + assert_eq!( + validate_vm_sandbox(&sandbox, false).unwrap_err().code(), + Code::FailedPrecondition + ); } #[test] diff --git a/crates/openshell-sdk/README.md b/crates/openshell-sdk/README.md index 001b2e9249..850200e110 100644 --- a/crates/openshell-sdk/README.md +++ b/crates/openshell-sdk/README.md @@ -212,3 +212,11 @@ These helpers use the raw client's authentication slot. They do not perform OIDC - Async-only. Tonic is async-native; callers needing a blocking call can wrap with their own runtime. - The curated surface will grow as more RPCs graduate from `raw`. + +## Next minor API migration + +This source-breaking API change targets the next minor release and must not be backported into the `0.1.x` patch series. + +Raw `HealthResponse`, public and compute resource requirements, resource capabilities and workload-template resource messages gain acknowledgement, enforcement and sampled-budget fields. Complete generated Rust literals must initialize the new fields or use `..Default::default()`. Complete TypeScript Health responses need `supportsSampledResourceBudget: false` or a generated message builder. + +Legacy wire fields retain their identities and new optional fields remain absent on old payloads. Before sending a budget-bearing write, check the Health acknowledgement for field preservation. The selected driver's admission separately checks enforcement. Upgrade gateway writers together before persisting budgets because older writers can discard them. diff --git a/crates/openshell-sdk/src/client.rs b/crates/openshell-sdk/src/client.rs index 8241ac49b5..2239ea9626 100644 --- a/crates/openshell-sdk/src/client.rs +++ b/crates/openshell-sdk/src/client.rs @@ -168,16 +168,21 @@ impl OpenShellClient { }) } + // Public Health acknowledges wire/admission support without requiring the + // elevated GetGatewayInfo permission. Driver support remains authoritative. + async fn require_sampled_budget_support(&self) -> Result<()> { + let health = self + .unary(|mut grpc| async move { grpc.health(proto::HealthRequest {}).await }) + .await?; + if !health.supports_sampled_resource_budget { + return Err(SdkError::InvalidConfig { message: "gateway does not acknowledge sampled resource budgets; upgrade it before creating this workload".into(), status: None }); + } + Ok(()) + } + /// Create a new sandbox from a curated [`SandboxSpec`]. pub async fn create_sandbox(&self, spec: SandboxSpec) -> Result { - let request = create_sandbox_request(spec); - let response = self - .unary(|mut grpc| { - let request = request.clone(); - async move { grpc.create_sandbox(request).await } - }) - .await?; - sandbox_from_create_response(response) + self.workspace("default").create_sandbox(spec).await } /// Create a new sandbox from a workspace-scoped workload template name. @@ -200,17 +205,9 @@ impl OpenShellClient { &self, template: SandboxWorkloadTemplate, ) -> Result { - let response = self - .unary(|mut grpc| { - let request = proto::CreateSandboxTemplateRequest { - request_id: String::new(), - template: Some(template.clone()), - workspace_scope: Some(proto::workspace_selector("default")), - }; - async move { grpc.create_sandbox_template(request).await } - }) - .await?; - sandbox_template_from_response(response.template) + self.workspace("default") + .create_sandbox_template(template) + .await } /// Fetch a reusable sandbox template by name from the default workspace. @@ -1015,6 +1012,9 @@ impl WorkspaceScopedClient { /// Create a new sandbox in this workspace. pub async fn create_sandbox(&self, spec: SandboxSpec) -> Result { + if spec.sampled_budget.is_some() { + self.client.require_sampled_budget_support().await?; + } let mut request = create_sandbox_request(spec); request.workspace_scope = Some(proto::workspace_selector(&self.workspace)); let response = self @@ -1049,6 +1049,16 @@ impl WorkspaceScopedClient { &self, template: SandboxWorkloadTemplate, ) -> Result { + if template + .spec + .as_ref() + .and_then(|s| s.workload.as_ref()) + .and_then(|w| w.resources.as_ref()) + .and_then(|r| r.sampled_budget.as_ref()) + .is_some() + { + self.client.require_sampled_budget_support().await?; + } let response = self .client .unary(|mut grpc| { @@ -1478,6 +1488,7 @@ fn create_sandbox_request(spec: SandboxSpec) -> proto::CreateSandboxRequest { environment, providers, gpu, + sampled_budget, command, tty, service_exposures, @@ -1487,9 +1498,11 @@ fn create_sandbox_request(spec: SandboxSpec) -> proto::CreateSandboxRequest { image, ..proto::SandboxTemplate::default() }); - let resource_requirements = gpu.then_some(proto::ResourceRequirements { - gpu: Some(proto::GpuResourceRequirements { count: None }), - }); + let resource_requirements = + (gpu || sampled_budget.is_some()).then_some(proto::ResourceRequirements { + gpu: gpu.then_some(proto::GpuResourceRequirements { count: None }), + sampled_budget, + }); proto::CreateSandboxRequest { request_id: String::new(), spec: Some(proto::SandboxSpec { diff --git a/crates/openshell-sdk/src/types.rs b/crates/openshell-sdk/src/types.rs index 571ee42b68..03f362e0eb 100644 --- a/crates/openshell-sdk/src/types.rs +++ b/crates/openshell-sdk/src/types.rs @@ -291,6 +291,8 @@ pub struct SandboxSpec { /// Request a GPU. Driver-specific device selection is configured via /// driver config on the raw proto surface (see [`crate::raw`]). pub gpu: bool, + /// Explicit sampled account-wide budgets, distinct from hard resource limits. + pub sampled_budget: Option, /// Exact canonical command. Empty selects the gateway's scratch login shell. pub command: Vec, /// Allocate a retained pseudo-terminal for the canonical command. diff --git a/crates/openshell-sdk/tests/client_mock.rs b/crates/openshell-sdk/tests/client_mock.rs index 1f3f791a08..e4b338bdc8 100644 --- a/crates/openshell-sdk/tests/client_mock.rs +++ b/crates/openshell-sdk/tests/client_mock.rs @@ -20,7 +20,7 @@ use openshell_sdk::{ }; use std::collections::HashMap; use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::time::Duration; use tokio::net::TcpListener; use tokio::sync::Mutex; @@ -48,6 +48,8 @@ fn selects_all_workspaces(scope: &Option>, last_get_workspace: Mutex>, last_create: Mutex>, @@ -312,6 +314,9 @@ impl OpenShell for TestOpenShell { &self, request: tonic::Request, ) -> Result, Status> { + if self.state.health_failure.load(Ordering::SeqCst) { + return Err(Status::unimplemented("legacy health RPC")); + } if let Some(expected) = &self.state.require_bearer { let got = request .metadata() @@ -325,6 +330,7 @@ impl OpenShell for TestOpenShell { Ok(Response::new(proto::HealthResponse { status: proto::ServiceStatus::Healthy.into(), version: "test-1.2.3".to_string(), + supports_sampled_resource_budget: self.state.sampled_budget_ack.load(Ordering::SeqCst), })) } @@ -1241,11 +1247,16 @@ async fn create_sandbox_passes_spec_through() { let mut labels = HashMap::new(); labels.insert("team".to_string(), "core".to_string()); + let budget = proto::sandbox::v1::SampledResourceBudget { + cpu_time: Some(Duration::from_secs(2).try_into().unwrap()), + resident_bytes: Some(4096), + }; let spec = SandboxSpec { name: Some("my-box".to_string()), image: Some("ghcr.io/foo:bar".to_string()), labels: labels.clone(), gpu: true, + sampled_budget: Some(budget), service_exposures: vec![ServiceExposure { service: "web".to_string(), target_port: 8080, @@ -1254,6 +1265,13 @@ async fn create_sandbox_passes_spec_through() { ..Default::default() }; + assert!(client.create_sandbox(spec.clone()).await.is_err()); + assert!(state.last_create.lock().await.is_none()); + state.health_failure.store(true, Ordering::SeqCst); + assert!(client.create_sandbox(spec.clone()).await.is_err()); + assert!(state.last_create.lock().await.is_none()); + state.health_failure.store(false, Ordering::SeqCst); + state.sampled_budget_ack.store(true, Ordering::SeqCst); let result = client.create_sandbox(spec).await.unwrap(); assert_eq!(result.name, "my-box"); assert_eq!(result.phase, SandboxPhase::Provisioning); @@ -1274,6 +1292,14 @@ async fn create_sandbox_passes_spec_through() { proto::ServiceAuthorizationMode::BearerPassthrough ); let observed_spec = observed.spec.unwrap(); + let budget = observed_spec + .resource_requirements + .as_ref() + .unwrap() + .sampled_budget + .unwrap(); + assert_eq!(budget.cpu_time.unwrap().seconds, 2); + assert_eq!(budget.resident_bytes, Some(4096)); assert!( observed_spec .resource_requirements @@ -1326,10 +1352,30 @@ async fn sandbox_template_crud_uses_default_workspace() { let endpoint = start_mock(state.clone()).await; let client = connect(&endpoint).await; - let created = client - .create_sandbox_template(workload_template_proto("python", "")) - .await - .unwrap(); + let mut template = workload_template_proto("python", ""); + template + .spec + .as_mut() + .unwrap() + .workload + .as_mut() + .unwrap() + .resources = Some(proto::SandboxResources { + sampled_budget: Some(proto::sandbox::v1::SampledResourceBudget { + cpu_time: None, + resident_bytes: Some(4096), + }), + ..Default::default() + }); + assert!( + client + .create_sandbox_template(template.clone()) + .await + .is_err() + ); + assert!(state.last_template_create.lock().await.is_none()); + state.sampled_budget_ack.store(true, Ordering::SeqCst); + let created = client.create_sandbox_template(template).await.unwrap(); assert_eq!(created.metadata.as_ref().unwrap().name, "python"); let observed_create = state.last_template_create.lock().await.clone().unwrap(); @@ -2105,8 +2151,15 @@ async fn workspace_scoped_create_passes_workspace() { let ws = client.workspace("staging"); let spec = SandboxSpec { name: Some("my-box".to_string()), + sampled_budget: Some(proto::sandbox::v1::SampledResourceBudget { + cpu_time: None, + resident_bytes: Some(4096), + }), ..Default::default() }; + assert!(ws.create_sandbox(spec.clone()).await.is_err()); + assert!(state.last_create.lock().await.is_none()); + state.sampled_budget_ack.store(true, Ordering::SeqCst); let result = ws.create_sandbox(spec).await.unwrap(); assert_eq!(result.name, "my-box"); @@ -2185,9 +2238,25 @@ async fn workspace_scoped_sandbox_template_crud_passes_workspace() { let client = connect(&endpoint).await; let ws = client.workspace("staging"); - ws.create_sandbox_template(workload_template_proto("python", "staging")) - .await - .unwrap(); + let mut template = workload_template_proto("python", "staging"); + template + .spec + .as_mut() + .unwrap() + .workload + .as_mut() + .unwrap() + .resources = Some(proto::SandboxResources { + sampled_budget: Some(proto::sandbox::v1::SampledResourceBudget { + cpu_time: None, + resident_bytes: Some(4096), + }), + ..Default::default() + }); + assert!(ws.create_sandbox_template(template.clone()).await.is_err()); + assert!(state.last_template_create.lock().await.is_none()); + state.sampled_budget_ack.store(true, Ordering::SeqCst); + ws.create_sandbox_template(template).await.unwrap(); let observed_create = state.last_template_create.lock().await.clone().unwrap(); assert_eq!( selected_workspace(&observed_create.workspace_scope), diff --git a/crates/openshell-server/src/compute/image_update.rs b/crates/openshell-server/src/compute/image_update.rs index 4a19ee2a53..018c33230e 100644 --- a/crates/openshell-server/src/compute/image_update.rs +++ b/crates/openshell-server/src/compute/image_update.rs @@ -398,7 +398,9 @@ impl ComputeRuntime { // Each sandbox owns independent locks even though these // futures are polled concurrently in the same Tokio task. crate::persistence::lock_order::branch(async move { - if let Err(error) = self.recover_one_sandbox_update(&id).await { + // Resource input growth makes this recovery future exceed the + // stack-size lint budget; keep the existing await on the heap. + if let Err(error) = Box::pin(self.recover_one_sandbox_update(&id)).await { tracing::warn!(sandbox_id = %id, %error, "Cannot read pending sandbox update for recovery"); } @@ -970,15 +972,16 @@ mod tests { .get_sandbox_image_state(original.object_id()) .await .unwrap(); - runtime - .recover_one_sandbox_update(original.object_id()) - .await - .unwrap(); - runtime - .clone() - .recover_one_sandbox_update(original.object_id()) + Box::pin(runtime.recover_one_sandbox_update(original.object_id())) .await .unwrap(); + Box::pin( + runtime + .clone() + .recover_one_sandbox_update(original.object_id()), + ) + .await + .unwrap(); assert_eq!(image_calls(&driver).len(), 2); assert_eq!( runtime @@ -994,8 +997,7 @@ mod tests { let mut restarted = runtime.clone(); restarted.update_recovery_backoff = Arc::default(); driver.set_image_update_error(None); - restarted - .recover_one_sandbox_update(original.object_id()) + Box::pin(restarted.recover_one_sandbox_update(original.object_id())) .await .unwrap(); assert_eq!(image_calls(&driver).len(), 3); @@ -1018,8 +1020,7 @@ mod tests { .unwrap_err(); // The original runtime still remembers the previous operation's delay, // but the new operation must be eligible immediately. - runtime - .recover_one_sandbox_update(original.object_id()) + Box::pin(runtime.recover_one_sandbox_update(original.object_id())) .await .unwrap(); assert_eq!(image_calls(&driver).len(), 5); @@ -1029,8 +1030,7 @@ mod tests { ); driver.set_image_update_error(None); - restarted - .recover_one_sandbox_update(original.object_id()) + Box::pin(restarted.recover_one_sandbox_update(original.object_id())) .await .unwrap(); runtime.recover_pending_sandbox_updates().await.unwrap(); diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index a6768040e6..b5e31370f4 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -691,7 +691,14 @@ struct RemoteComputeDriver { client: RemoteComputeDriverClient, app_endpoints_client: openshell_core::proto::compute::v1::compute_driver_app_endpoints_client::ComputeDriverAppEndpointsClient>, configuration_updates_client: openshell_core::proto::compute::v1::compute_driver_configuration_updates_client::ComputeDriverConfigurationUpdatesClient>, - admission_acknowledgement: Arc>>, + admission_handshake: Arc>>, +} + +#[derive(Debug, Clone)] +struct RemoteAdmissionHandshake { + policy: String, + resource_contract: bool, + resources: Option, } type RemoteComputeDriverClient = ComputeDriverClient< @@ -704,7 +711,7 @@ impl RemoteComputeDriver { client: ComputeDriverClient::with_interceptor(channel.clone(), TraceContextInterceptor), app_endpoints_client: openshell_core::proto::compute::v1::compute_driver_app_endpoints_client::ComputeDriverAppEndpointsClient::with_interceptor(channel.clone(), TraceContextInterceptor), configuration_updates_client: openshell_core::proto::compute::v1::compute_driver_configuration_updates_client::ComputeDriverConfigurationUpdatesClient::with_interceptor(channel, TraceContextInterceptor), - admission_acknowledgement: Arc::new(StdMutex::new(None)), + admission_handshake: Arc::new(StdMutex::new(None)), } } @@ -714,7 +721,7 @@ impl RemoteComputeDriver { async fn verify_admission_policy(&self, external_isolation: bool) -> Result<(), Status> { let expected = self - .admission_acknowledgement + .admission_handshake .lock() .map_err(|_| Status::internal("admission handshake lock poisoned"))? .clone() @@ -735,15 +742,36 @@ impl RemoteComputeDriver { "remote compute driver no longer supports isolation backend registration; restart the gateway after upgrading the driver", )); } - if current.resource_admission_policy != expected { + if current.resource_admission_policy != expected.policy { return Err(Status::failed_precondition( "remote driver admission policy changed; restart gateway after configuring matching policy", )); } + verify_resource_contract(expected.resource_contract, &expected.resources, ¤t)?; Ok(()) } } +fn verify_resource_contract( + negotiated: bool, + resources: &Option, + current: &openshell_core::proto::compute::v1::GetCapabilitiesResponse, +) -> Result<(), Status> { + if negotiated + && (!current.extension.as_ref().is_some_and(|metadata| { + metadata + .supported_capabilities + .iter() + .any(|cap| cap == openshell_core::extension_protocol::COMPUTE_RESOURCE_CONTRACT) + }) || ¤t.resource_capabilities != resources) + { + return Err(Status::failed_precondition( + "compute resource contract changed; restart the gateway after updating the driver", + )); + } + Ok(()) +} + #[tonic::async_trait] impl openshell_core::compute::ComputeDriver for RemoteComputeDriver { async fn update_sandbox_configuration( @@ -772,10 +800,18 @@ impl ComputeDriver for RemoteComputeDriver { { let mut client = self.client(); let response = client.get_capabilities(request).await?; - self.admission_acknowledgement + self.admission_handshake .lock() .map_err(|_| Status::internal("admission handshake lock poisoned"))? - .get_or_insert_with(|| response.get_ref().resource_admission_policy.clone()); + .get_or_insert_with(|| RemoteAdmissionHandshake { + policy: response.get_ref().resource_admission_policy.clone(), + resource_contract: response.get_ref().extension.as_ref().is_some_and(|e| { + e.supported_capabilities + .iter() + .any(|c| c == openshell_core::extension_protocol::COMPUTE_RESOURCE_CONTRACT) + }), + resources: response.get_ref().resource_capabilities, + }); Ok(response) } @@ -1296,9 +1332,52 @@ impl ComputeRuntime { if let Some(spec) = driver_sandbox.spec.as_mut() { spec.isolation_backend.clone_from(&self.isolation_backend); } + openshell_core::resource_contract::validate_request( + &driver_sandbox, + self.driver_info.resource_capabilities.as_ref(), + self.driver_info + .negotiated_extension + .supported_capabilities + .iter() + .any(|cap| cap == openshell_core::extension_protocol::COMPUTE_RESOURCE_CONTRACT), + ) + .map_err(Status::failed_precondition)?; Ok(driver_sandbox) } + /// Verify current guarantees before claiming lifecycle work or consuming staging. + async fn admit_driver_sandbox_for_launch( + &self, + sandbox: &Sandbox, + ) -> Result { + let projected = self.driver_sandbox_for_launch(sandbox)?; + let negotiated = self + .driver_info + .negotiated_extension + .supported_capabilities + .iter() + .any(|cap| cap == openshell_core::extension_protocol::COMPUTE_RESOURCE_CONTRACT); + if negotiated { + let current = self + .driver + .call( + openshell_otel::rpc::GET_CAPABILITIES, + Some(sandbox.object_id()), + |driver| async move { + driver + .get_capabilities(Request::new(GetCapabilitiesRequest { + gateway: Some(gateway_metadata(ExtensionFamily::Compute)), + })) + .await + }, + ) + .await? + .into_inner(); + verify_resource_contract(true, &self.driver_info.resource_capabilities, ¤t)?; + } + Ok(projected) + } + fn driver_start_request( &self, sandbox: &Sandbox, @@ -1492,7 +1571,7 @@ impl ComputeRuntime { .as_ref() .and_then(|spec| spec.template.as_ref()), )?; - let mut driver_sandbox = self.driver_sandbox_for_launch(sandbox)?; + let mut driver_sandbox = self.admit_driver_sandbox_for_launch(sandbox).await?; if let Some(spec) = driver_sandbox.spec.as_mut() { spec.await_main_process_attachment = await_main_process_attachment; } @@ -1657,6 +1736,7 @@ impl ComputeRuntime { .as_ref() .and_then(|spec| spec.template.as_ref()), )?; + self.admit_driver_sandbox_for_launch(&sandbox).await?; let sandbox_id = sandbox.object_id().to_string(); let mut sandbox = sandbox; let metadata = sandbox @@ -2437,6 +2517,7 @@ impl ComputeRuntime { if phase == SandboxPhase::Ready { return Ok(current); } + self.admit_driver_sandbox_for_launch(¤t).await?; let provisioning_timeout = phase == SandboxPhase::Error && provisioning_deadline::timed_out(¤t); if provisioning_timeout @@ -4325,6 +4406,11 @@ impl ComputeRuntime { continue; } + if let Err(error) = self.admit_driver_sandbox_for_launch(&sandbox).await { + warn!(sandbox_id, %error, "Rejected changed recovered resource guarantees"); + failed += 1; + continue; + } if let Err(error) = sandbox_runtime_generation(&sandbox) { warn!(sandbox_id, %error, "Persisted sandbox runtime identity is invalid"); authentication_failed(sandbox.object_id()); @@ -4597,6 +4683,10 @@ impl ComputeRuntime { .await; continue; } + if let Err(error) = self.admit_driver_sandbox_for_launch(&sandbox).await { + warn!(%error, "Rejected changed pending resource guarantees"); + continue; + } let sandbox_id = sandbox.object_id().to_string(); let expected_runtime_identity = sandbox_compute_runtime_identity(&sandbox); let launch_authentication = if self.isolation_backend.is_some() { @@ -5081,6 +5171,9 @@ impl ComputeRuntime { let sandbox_name = current.object_name().to_string(); self.validate_isolation_backend_binding(¤t) .map_err(|error| error.to_string())?; + self.admit_driver_sandbox_for_launch(¤t) + .await + .map_err(|error| error.to_string())?; let already_claimed = current .status .as_ref() @@ -7209,6 +7302,7 @@ fn driver_sandbox_spec_from_public( policy: spec.policy.clone(), resource_requirements: spec.resource_requirements.as_ref().map(|requirements| { DriverSandboxResourceRequirements { + sampled_budget: requirements.sampled_budget, gpu: requirements .gpu .as_ref() @@ -8875,6 +8969,7 @@ mod tests { let public = SandboxSpec { resource_requirements: Some(openshell_core::proto::ResourceRequirements { gpu: Some(openshell_core::proto::GpuResourceRequirements { count: Some(2) }), + ..Default::default() }), ..Default::default() }; @@ -12831,6 +12926,7 @@ mod tests { Some(&SandboxSpec { resource_requirements: Some(openshell_core::proto::ResourceRequirements { gpu: Some(openshell_core::proto::GpuResourceRequirements { count: None }), + ..Default::default() }), ..Default::default() }), @@ -17001,6 +17097,7 @@ mod tests { spec: Some(SandboxSpec { resource_requirements: Some(openshell_core::proto::ResourceRequirements { gpu: Some(openshell_core::proto::GpuResourceRequirements { count: None }), + ..Default::default() }), ..Default::default() }), @@ -18729,61 +18826,213 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" ); } + #[tokio::test] + #[cfg(unix)] + async fn retained_resource_refusal_precedes_lifecycle_claim_and_stop() { + use crate::test_support::{FakeComputeDriver, FakeComputeDriverCall}; + use openshell_core::proto::sandbox::v1::{ + ResourceEnforcement, ResourceEnforcementKind, ResourceEnforcementScope, + }; + for remove_contract in [false, true] { + for mode in 0..4 { + let directory = tempfile::tempdir().unwrap(); + let socket = directory.path().join("resource-drift.sock"); + let driver = FakeComputeDriver::new().with_gateway_manages_lifecycle(); + driver.set_resource_contract( + true, + Some(DriverResourceCapabilities { + cpu: Some( + openshell_core::proto::compute::v1::CpuResourceCapabilities { + limit_supported: true, + limit_enforcement: Some(ResourceEnforcement { + kind: ResourceEnforcementKind::KernelLimit as i32, + scope: ResourceEnforcementScope::Container as i32, + }), + ..Default::default() + }, + ), + ..Default::default() + }), + ); + let _server = driver.serve_uds(&socket).unwrap(); + let endpoint = connect_remote_compute_driver("external-test", &socket) + .await + .unwrap(); + let store = Arc::new(crate::persistence::test_store().await); + crate::ensure_default_workspace(&store).await.unwrap(); + let runtime = ComputeRuntime::new_remote_driver( + endpoint, + store, + SandboxIndex::new(), + SandboxWatchBus::new(), + TracingLogBus::new(), + Arc::new(SupervisorSessionRegistry::new()), + ) + .await + .unwrap(); + driver.set_resource_contract(!remove_contract, None); + driver.clear_calls(); + let mut sandbox = sandbox_record("retained", "retained", SandboxPhase::Stopped); + sandbox.spec = Some(SandboxSpec { + template: Some(SandboxTemplate { + resources: Some( + openshell_core::proto_struct::json_object_to_struct( + serde_json::json!({"limits": {"cpu": "1"}}) + .as_object() + .unwrap() + .clone(), + ) + .unwrap(), + ), + ..Default::default() + }), + restart_policy: SandboxRestartPolicy::Always as i32, + ..Default::default() + }); + if mode == 1 { + apply_main_process_restart( + &mut sandbox, + "previous-main", + 9, + openshell_core::time::now_ms() + .saturating_sub(RESTART_TERMINAL_DELIVERY_GRACE_MS + 1), + ); + } + if mode == 2 { + sandbox.set_phase(SandboxPhase::Ready as i32); + } + if mode == 3 { + sandbox.set_phase(SandboxPhase::Starting as i32); + } + runtime.store.put_message(&sandbox).await.unwrap(); + let before = runtime + .store + .get_message::("retained") + .await + .unwrap() + .unwrap(); + match mode { + 0 => { + let error = runtime + .start_sandbox("default", "retained") + .await + .unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("resource contract changed")); + } + 1 => { + let error = runtime + .restart_sandbox_runtime("retained") + .await + .unwrap_err(); + assert!(error.contains("resource contract changed"), "{error}"); + } + 2 => { + runtime + .start_persisted_sandboxes_with_authentication( + |_| async { + panic!( + "changed guarantees must precede recovered authentication" + ) + }, + |_| async { Ok(()) }, + |_| {}, + ) + .await + .unwrap(); + } + _ => { + runtime + .recover_persisted_lifecycle_transitions() + .await + .unwrap(); + } + } + let after = runtime + .store + .get_message::("retained") + .await + .unwrap() + .unwrap(); + assert_eq!( + after, before, + "refusal must preserve generation, lifecycle, and restart schedule" + ); + assert_eq!( + driver.calls(), + vec![FakeComputeDriverCall::GetCapabilities], + "current guarantees must be checked before driver STOP/START" + ); + } + } + } + #[tokio::test] #[cfg(unix)] async fn remote_compute_driver_rejects_policy_changes_before_forwarding() { use crate::test_support::{FakeComputeDriver, FakeComputeDriverCall}; - let dir = tempfile::tempdir().unwrap(); - let socket_path = dir.path().join("admission.sock"); - let driver = FakeComputeDriver::new(); - let _server = driver.serve_uds(&socket_path).unwrap(); - let endpoint = connect_remote_compute_driver("external-test", &socket_path) - .await - .unwrap(); - let remote = RemoteComputeDriver::new(endpoint.channel); - remote - .get_capabilities(Request::new(GetCapabilitiesRequest { - gateway: Some(gateway_metadata(ExtensionFamily::Compute)), - })) - .await - .unwrap(); - driver.set_admission_acknowledgement(String::new()); - driver.clear_calls(); - let sandbox = DriverSandbox::default(); - let status = remote - .validate_sandbox_create(Request::new(ValidateSandboxCreateRequest { - sandbox: Some(sandbox.clone()), - })) - .await - .unwrap_err(); - assert_eq!(status.code(), Code::FailedPrecondition); - assert!( + for change in 0..3 { + let dir = tempfile::tempdir().unwrap(); + let socket_path = dir.path().join("admission.sock"); + let driver = FakeComputeDriver::new(); + driver.set_resource_contract(true, Some(DriverResourceCapabilities::default())); + let _server = driver.serve_uds(&socket_path).unwrap(); + let endpoint = connect_remote_compute_driver("external-test", &socket_path) + .await + .unwrap(); + let remote = RemoteComputeDriver::new(endpoint.channel); remote - .create_sandbox(Request::new(CreateSandboxRequest { - sandbox: Some(sandbox) + .get_capabilities(Request::new(GetCapabilitiesRequest { + gateway: Some(gateway_metadata(ExtensionFamily::Compute)), })) .await - .is_err() - ); - assert!( - remote - .start_sandbox(Request::new(StartSandboxRequest::default())) + .unwrap(); + match change { + 0 => driver.set_admission_acknowledgement(String::new()), + 1 => { + driver + .set_resource_contract(false, Some(DriverResourceCapabilities::default())); + } + _ => driver.set_resource_contract(true, None), + } + driver.clear_calls(); + let sandbox = DriverSandbox::default(); + let status = remote + .validate_sandbox_create(Request::new(ValidateSandboxCreateRequest { + sandbox: Some(sandbox.clone()), + })) .await - .is_err() - ); - let status = openshell_core::compute::ComputeDriver::update_sandbox_configuration( - &remote, - Request::new( - openshell_core::proto::compute::v1::UpdateSandboxConfigurationRequest::default(), - ), - ) - .await - .unwrap_err(); - assert_eq!(status.code(), Code::FailedPrecondition); - assert_eq!( - driver.calls(), - vec![FakeComputeDriverCall::GetCapabilities; 4] - ); + .unwrap_err(); + assert_eq!(status.code(), Code::FailedPrecondition); + assert!( + remote + .create_sandbox(Request::new(CreateSandboxRequest { + sandbox: Some(sandbox) + })) + .await + .is_err() + ); + assert!( + remote + .start_sandbox(Request::new(StartSandboxRequest::default())) + .await + .is_err() + ); + let status = openshell_core::compute::ComputeDriver::update_sandbox_configuration( + &remote, + Request::new( + openshell_core::proto::compute::v1::UpdateSandboxConfigurationRequest::default( + ), + ), + ) + .await + .unwrap_err(); + assert_eq!(status.code(), Code::FailedPrecondition); + assert_eq!( + driver.calls(), + vec![FakeComputeDriverCall::GetCapabilities; 4] + ); + } } #[tokio::test] diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index 69208348de..e44838fbad 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -268,6 +268,7 @@ impl OpenShell for OpenShellService { Ok(Response::new(HealthResponse { status: ServiceStatus::Healthy.into(), version: openshell_core::VERSION.to_string(), + supports_sampled_resource_budget: true, })) } @@ -977,14 +978,19 @@ fn public_resource_capabilities( resources: openshell_core::proto::compute::v1::ResourceCapabilities, ) -> ResourceCapabilities { ResourceCapabilities { + sampled_budget: resources.sampled_budget, cpu: resources.cpu.as_ref().map(|cpu| CpuResourceCapabilities { limit_supported: cpu.limit_supported, + limit_enforcement: cpu.limit_enforcement, + request_enforcement: cpu.request_enforcement, }), memory: resources .memory .as_ref() .map(|memory| MemoryResourceCapabilities { limit_supported: memory.limit_supported, + limit_enforcement: memory.limit_enforcement, + request_enforcement: memory.request_enforcement, }), gpu: resources.gpu.as_ref().map(|gpu| GpuResourceCapabilities { default_selection_supported: gpu.default_selection_supported, @@ -1323,19 +1329,33 @@ mod tests { let driver_capabilities = DriverResourceCapabilities { cpu: Some(DriverCpuResourceCapabilities { limit_supported: true, + limit_enforcement: Some(openshell_core::proto::sandbox::v1::ResourceEnforcement { + kind: openshell_core::proto::sandbox::v1::ResourceEnforcementKind::KernelLimit + as i32, + scope: openshell_core::proto::sandbox::v1::ResourceEnforcementScope::Container + as i32, + }), + ..Default::default() }), memory: Some(DriverMemoryResourceCapabilities { limit_supported: false, + ..Default::default() }), gpu: Some(DriverGpuResourceCapabilities { default_selection_supported: true, count_selection_supported: true, }), + sampled_budget: None, }; let capabilities = public_resource_capabilities(driver_capabilities); - assert!(capabilities.cpu.expect("CPU capabilities").limit_supported); + let cpu = capabilities.cpu.expect("CPU capabilities"); + assert!(cpu.limit_supported); + assert_eq!( + cpu.limit_enforcement.unwrap().scope, + openshell_core::proto::sandbox::v1::ResourceEnforcementScope::Container as i32 + ); assert!( !capabilities .memory diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 86ff12b170..735b429fed 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -981,15 +981,20 @@ fn sandbox_spec_from_workload_template( driver_config: spec.driver_config.clone(), ..SandboxTemplate::default() }), - resource_requirements: resources.and_then(template_gpu_requirements), + resource_requirements: resources.and_then(template_typed_resource_requirements), ..SandboxSpec::default() }) } -fn template_gpu_requirements(resources: &SandboxResources) -> Option { - Some(ResourceRequirements { - gpu: Some(resources.gpu?), - }) +fn template_typed_resource_requirements( + resources: &SandboxResources, +) -> Option { + (resources.gpu.is_some() || resources.sampled_budget.is_some()).then_some( + ResourceRequirements { + gpu: resources.gpu, + sampled_budget: resources.sampled_budget, + }, + ) } fn template_resource_struct(resources: &SandboxResources) -> Option { @@ -4306,6 +4311,158 @@ mod tests { GpuResourceRequirements, SandboxServiceExposure, ServiceAuthorizationMode, ServiceEndpoint, }; + #[tokio::test] + async fn sampled_resource_contract_is_enforced_before_create() { + use crate::test_support::{FakeComputeDriver, FakeComputeDriverCall}; + use openshell_core::extension_protocol::{ + COMPUTE_RESOURCE_CONTRACT, ExtensionFamily, extension_metadata, + }; + use openshell_core::proto::compute::v1::{GetCapabilitiesResponse, ResourceCapabilities}; + use openshell_core::proto::sandbox::v1::{ + ResourceEnforcementScope, SampledResourceBudget, SampledResourceBudgetCapabilities, + }; + let budget = SampledResourceBudget { + cpu_time: Some(prost_types::Duration { + seconds: 2, + nanos: 0, + }), + resident_bytes: Some(512 * 1024 * 1024), + }; + let caps = ResourceCapabilities { + sampled_budget: Some(SampledResourceBudgetCapabilities { + cpu_time_supported: true, + resident_bytes_supported: true, + scope: ResourceEnforcementScope::SandboxAccount as i32, + sampling_interval: Some(prost_types::Duration { + seconds: 0, + nanos: 100_000_000, + }), + terminate_on_excess: true, + cpu_time_resolution: Some(prost_types::Duration { + seconds: 0, + nanos: 1_000_000, + }), + per_process_cpu_limit: true, + }), + ..Default::default() + }; + for (modern, capabilities, cpu, memory, sampled, accepted) in [ + (true, Some(caps), "", "", Some(budget), true), + (true, Some(caps), "1", "", None, false), + (true, Some(caps), "", "512Mi", None, false), + (true, Some(caps), "1", "", Some(budget), false), + (true, None, "", "", Some(budget), false), + (false, Some(caps), "", "", Some(budget), false), + (false, None, "1", "", None, true), + ] { + let mut state = test_server_state().await; + let driver = Arc::new( + FakeComputeDriver::new().with_capabilities(GetCapabilitiesResponse { + driver_name: "test".into(), + default_image: "test/image".into(), + resource_capabilities: capabilities, + resource_admission_policy: + openshell_core::resource_admission::DriverAdmissionConfig::default() + .acknowledgement(), + extension: Some(extension_metadata( + ExtensionFamily::Compute, + "openshell/test", + "test", + if modern { + vec![COMPUTE_RESOURCE_CONTRACT.into()] + } else { + vec![] + }, + )), + ..Default::default() + }), + ); + let compute = crate::compute::ComputeRuntime::from_driver( + "test".into(), + driver.clone(), + None, + state.store.clone(), + crate::sandbox_index::SandboxIndex::new(), + crate::sandbox_watch::SandboxWatchBus::new(), + crate::tracing_bus::TracingLogBus::new(), + Arc::new(crate::supervisor_session::SupervisorSessionRegistry::new()), + ) + .await + .unwrap(); + Arc::get_mut(&mut state).unwrap().compute = compute; + // Exercise the typed workload-template path as well as public-to-driver mapping. + let resources = SandboxResources { + cpu: cpu.into(), + memory: memory.into(), + sampled_budget: sampled, + ..Default::default() + }; + let spec = SandboxSpec { + template: Some(SandboxTemplate { + resources: template_resource_struct(&resources), + ..Default::default() + }), + resource_requirements: template_typed_resource_requirements(&resources), + policy: Some(openshell_policy::restrictive_default_policy()), + ..Default::default() + }; + let result = handle_create_sandbox( + &state, + authed_request(CreateSandboxRequest { + name: "budget".into(), + spec: Some(spec), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await; + assert_eq!( + result.is_ok(), + accepted, + "modern={modern}, cpu={cpu}, memory={memory}" + ); + let requests: Vec<_> = driver + .calls() + .into_iter() + .filter_map(|call| match call { + FakeComputeDriverCall::ValidateSandboxCreate { sandbox } + | FakeComputeDriverCall::CreateSandbox { sandbox } => Some(sandbox.unwrap()), + _ => None, + }) + .collect(); + assert_eq!(requests.len(), if accepted { 2 } else { 0 }); + let stored = state + .store + .get_message_by_name::("default", "budget") + .await + .unwrap(); + assert_eq!(stored.is_some(), accepted); + if accepted && sampled.is_some() { + for request in requests { + assert_eq!( + request + .spec + .unwrap() + .resource_requirements + .unwrap() + .sampled_budget, + sampled + ); + } + assert_eq!( + stored + .unwrap() + .spec + .unwrap() + .resource_requirements + .unwrap() + .sampled_budget, + sampled + ); + } + } + } + #[tokio::test] async fn missing_launch_signer_precedes_driver_validation_and_preserves_staged_archive() { let directory = tempfile::tempdir().unwrap(); @@ -4470,6 +4627,7 @@ mod tests { policy: Some(openshell_core::proto::SandboxPolicy::default()), resource_requirements: Some(ResourceRequirements { gpu: Some(GpuResourceRequirements { count: Some(1) }), + ..Default::default() }), ..SandboxSpec::default() }), @@ -5200,6 +5358,7 @@ mod tests { cpu: "2".to_string(), memory: "4Gi".to_string(), gpu: Some(GpuResourceRequirements { count: Some(1) }), + ..Default::default() }), }), driver_config: None, diff --git a/crates/openshell-server/src/grpc/validation.rs b/crates/openshell-server/src/grpc/validation.rs index d84488bcca..aaf8f5b28e 100644 --- a/crates/openshell-server/src/grpc/validation.rs +++ b/crates/openshell-server/src/grpc/validation.rs @@ -312,6 +312,15 @@ fn validate_main_process_command(command: &[String]) -> Result<(), Status> { } fn validate_gpu_request_fields(spec: &SandboxSpec) -> Result<(), Status> { + if let Some(budget) = spec + .resource_requirements + .as_ref() + .and_then(|r| r.sampled_budget.as_ref()) + { + openshell_core::resource_contract::validate_sampled_budget(budget).map_err(|error| { + invalid_argument("spec.resource_requirements.sampled_budget", error) + })?; + } if openshell_core::gpu::sandbox_gpu_count(spec.resource_requirements.as_ref()) == Some(0) { return Err(invalid_argument( "spec.resource_requirements.gpu.count", @@ -1242,6 +1251,7 @@ mod tests { let spec = SandboxSpec { resource_requirements: Some(openshell_core::proto::ResourceRequirements { gpu: Some(openshell_core::proto::GpuResourceRequirements { count: None }), + ..Default::default() }), ..Default::default() }; @@ -1253,6 +1263,7 @@ mod tests { let spec = SandboxSpec { resource_requirements: Some(openshell_core::proto::ResourceRequirements { gpu: Some(openshell_core::proto::GpuResourceRequirements { count: Some(2) }), + ..Default::default() }), ..Default::default() }; @@ -1264,6 +1275,7 @@ mod tests { let spec = SandboxSpec { resource_requirements: Some(openshell_core::proto::ResourceRequirements { gpu: Some(openshell_core::proto::GpuResourceRequirements { count: Some(0) }), + ..Default::default() }), ..Default::default() }; diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index d8da420871..cc12fc8bc9 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -178,11 +178,11 @@ mod tests { // image wire identity, now with explicit presence. Compute configuration // updates use a separate capability/service and add one classified RPC. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "3b4c2af6cf22e7ed1d84279ff865acb0db7199717bee9bc041caef791334104d"; + "5ef822bc5817a921f3f006c8ff6ee30ceffdb81803f929c84ab11a1465fbf1dc"; const DURABLE_SCHEMA_SHA256: &str = - "cb15f8f79a855064a43fe8c6cd908bf07eca6604e2bf8ab7d2de0ef33e97da4e"; + "3716f9d2b96cdfc2dc92d6825a3729e3d695d7d7e1dd84733369b5c38dc91342"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = - "c0c3ea17f105aaa6d843c466fd4240451c2105dce7675384935d4aaecdac8abf"; + "a2ce7f9483846f81e024bdf31dc944bdd109aaaae294d8b3c58eac3ca5079f83"; // Encoded with the schema that omitted stable_placeholder. Keep these // bytes fixed so the current encoder cannot hide a compatibility change. const LEGACY_STATIC_PROFILE: &str = "0a240a116c65676163792d70726f66696c652d6964120d6c65676163792d737461746963280712510a0d6c65676163792d73746174696312184c6567616379207374617469632063726564656e7469616c2a260a076170695f6b65791a1153594e5448455449435f4150495f4b455920012a06626561726572"; @@ -723,9 +723,9 @@ mod tests { overlap_hash.as_str(), ), ( - (337, 31), - (100, 22), - (86, 22), + (340, 33), + (101, 22), + (87, 22), PUBLIC_RPC_SCHEMA_SHA256, DURABLE_SCHEMA_SHA256, PUBLIC_DURABLE_OVERLAP_SHA256 @@ -786,6 +786,11 @@ mod tests { assert!(!crate::policy_store::permits_initial_static_policy_repair( &sandbox )); + let resources = openshell_core::proto::ResourceRequirements::decode(&[10, 0][..]).unwrap(); + assert!(resources.gpu.is_some()); + assert!(resources.sampled_budget.is_none()); + let health = openshell_core::proto::HealthResponse::decode(&[8, 1][..]).unwrap(); + assert!(!health.supports_sampled_resource_budget); assert_eq!(sandbox.encode_to_vec(), bytes); } diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs index b1d5736a93..e7a17f8be5 100644 --- a/crates/openshell-server/src/test_support.rs +++ b/crates/openshell-server/src/test_support.rs @@ -267,6 +267,22 @@ impl FakeComputeDriver { self.with_state(|state| state.calls.clear()); } + pub fn set_resource_contract( + &self, + supported: bool, + resources: Option, + ) { + self.with_state(|state| { + let metadata = state.capabilities.extension.as_mut().unwrap(); + let capability = openshell_core::extension_protocol::COMPUTE_RESOURCE_CONTRACT; + metadata.supported_capabilities.retain(|c| c != capability); + if supported { + metadata.supported_capabilities.push(capability.into()); + } + state.capabilities.resource_capabilities = resources; + }); + } + pub fn set_admission_acknowledgement(&self, acknowledgement: String) { self.with_state(|state| state.capabilities.resource_admission_policy = acknowledgement); } diff --git a/crates/openshell-server/tests/common/mod.rs b/crates/openshell-server/tests/common/mod.rs index dd91664d39..032c172968 100644 --- a/crates/openshell-server/tests/common/mod.rs +++ b/crates/openshell-server/tests/common/mod.rs @@ -124,6 +124,7 @@ impl OpenShell for TestOpenShell { Ok(Response::new(HealthResponse { status: ServiceStatus::Healthy.into(), version: "test".to_string(), + supports_sampled_resource_budget: false, })) } diff --git a/docs/extensibility/drivers.mdx b/docs/extensibility/drivers.mdx index 67f9cd7b51..e16e43cb6c 100644 --- a/docs/extensibility/drivers.mdx +++ b/docs/extensibility/drivers.mdx @@ -63,6 +63,42 @@ active driver and negotiated capabilities. See the [Extension Driver configuration reference](/how-it-works/gateways/configuration#extension-driver) for configuration details. +### Resource Guarantees + +`openshell gateway info` reports resource enforcement for the configured compute +driver. CPU-rate and hard memory limits require aggregate kernel enforcement; +scheduler requests describe placement admission, not dedicated physical capacity. +Docker and Podman enforce container limits and reject reservations. Kubernetes +supports container limits and scheduler requests. VM and MXC reject these +portable requests until their drivers can enforce them. + +Drivers advertising `openshell.compute.resource-contract` must describe the +mechanism and scope of accepted limits and requests. Missing, unknown, or +per-process enforcement cannot satisfy an aggregate hard limit. Legacy drivers +retain their existing behavior and report absent descriptors as unreported. + +Sampled budgets are a separate opt-in request. For a driver that supports them: + +```shell +openshell sandbox template create bounded-worker \ + --cpu-time-budget 30s \ + --resident-memory-budget-bytes 1073741824 +openshell sandbox create --template bounded-worker +``` + +The descriptor reports account-wide observation, the interval between completed +samples, termination on excess, CPU-time resolution, and any companion +per-process CPU limit. Collection adds latency; short-lived processes can escape +observation and overshoot has no fixed bound. These budgets do not satisfy a +hard CPU-rate or memory quota. First-party container and VM drivers currently +reject sampled budgets; external drivers may provide them. + +Raw API clients must check `HealthResponse.supports_sampled_resource_budget` +before sending sampled budgets. The CLI and curated SDKs check it automatically. +This confirms that the gateway preserves budget fields; driver support is +reported separately. Upgrade all gateway replicas before using sampled budgets, +because older versions can discard budget fields. + ## Credential Drivers Credential drivers store provider credentials behind opaque handles. The diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index 88d090fc18..13bf34b022 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -179,16 +179,23 @@ message ResourceCapabilities { CpuResourceCapabilities cpu = 1; MemoryResourceCapabilities memory = 2; GpuResourceCapabilities gpu = 3; + openshell.sandbox.v1.SampledResourceBudgetCapabilities sampled_budget = 4; } message CpuResourceCapabilities { // The driver accepts and enforces a portable CPU limit. bool limit_supported = 1; + // Required for a negotiated resource contract. Absence means unreported. + openshell.sandbox.v1.ResourceEnforcement limit_enforcement = 2; + openshell.sandbox.v1.ResourceEnforcement request_enforcement = 3; } message MemoryResourceCapabilities { // The driver accepts and enforces a portable memory limit. bool limit_supported = 1; + // Required for a negotiated resource contract. Absence means unreported. + openshell.sandbox.v1.ResourceEnforcement limit_enforcement = 2; + openshell.sandbox.v1.ResourceEnforcement request_enforcement = 3; } message GpuResourceCapabilities { @@ -325,6 +332,8 @@ message DriverFenceEvidence { message ResourceRequirements { // GPU requirements for the sandbox. Presence indicates a GPU request. GpuResourceRequirements gpu = 1; + // Explicit opt-in to weaker sampled budgets; never inferred from hard limits. + openshell.sandbox.v1.SampledResourceBudget sampled_budget = 4; } // Driver GPU resource requirements. diff --git a/proto/openshell.proto b/proto/openshell.proto index b40ec68263..af5dd80056 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -936,6 +936,10 @@ message HealthResponse { // Service version. string version = 2; + // Gateway acknowledgement that sampled budget fields survive public writes + // and receive compute admission. This does not promise driver support. + // Budget-bearing clients must require true before writes to older gateways. + bool supports_sampled_resource_budget = 3; } // Current-user request. The identity comes from the authenticated request. @@ -1030,16 +1034,23 @@ message ResourceCapabilities { CpuResourceCapabilities cpu = 1; MemoryResourceCapabilities memory = 2; GpuResourceCapabilities gpu = 3; + openshell.sandbox.v1.SampledResourceBudgetCapabilities sampled_budget = 4; } message CpuResourceCapabilities { // The driver accepts and enforces a portable CPU limit. bool limit_supported = 1; + // Required for a negotiated resource contract. Absence means unreported. + openshell.sandbox.v1.ResourceEnforcement limit_enforcement = 2; + openshell.sandbox.v1.ResourceEnforcement request_enforcement = 3; } message MemoryResourceCapabilities { // The driver accepts and enforces a portable memory limit. bool limit_supported = 1; + // Required for a negotiated resource contract. Absence means unreported. + openshell.sandbox.v1.ResourceEnforcement limit_enforcement = 2; + openshell.sandbox.v1.ResourceEnforcement request_enforcement = 3; } message GpuResourceCapabilities { @@ -1115,6 +1126,8 @@ message SandboxSpec { message ResourceRequirements { // GPU requirements for the sandbox. Presence indicates a GPU request. GpuResourceRequirements gpu = 1; + // Explicit opt-in to weaker sampled budgets; never inferred from hard limits. + openshell.sandbox.v1.SampledResourceBudget sampled_budget = 4; } // Public GPU resource requirements. @@ -1200,6 +1213,7 @@ message SandboxResources { // request. When count is omitted, the request uses the selected driver's // default GPU assignment behavior. GpuResourceRequirements gpu = 3; + openshell.sandbox.v1.SampledResourceBudget sampled_budget = 4; } message SandboxServiceLevel { diff --git a/proto/sandbox.proto b/proto/sandbox.proto index b57c5d4cda..149803a491 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -548,3 +548,48 @@ enum SupervisorMiddlewareBinding { SUPERVISOR_MIDDLEWARE_BINDING_HTTP_RESPONSE_PRE_RETURN = 2; SUPERVISOR_MIDDLEWARE_BINDING_WEBSOCKET_MESSAGE_PRE_CREDENTIALS = 3; } + +// Explicit budgets observed across a sandbox account. These are not CPU-rate +// quotas or hard memory limits. Empty budgets are invalid. +message SampledResourceBudget { + google.protobuf.Duration cpu_time = 1; + optional uint64 resident_bytes = 2; +} + +// Mechanism and aggregation boundary for a resource request. +message ResourceEnforcement { + ResourceEnforcementKind kind = 1; + ResourceEnforcementScope scope = 2; +} + +enum ResourceEnforcementKind { + RESOURCE_ENFORCEMENT_KIND_UNSPECIFIED = 0; + RESOURCE_ENFORCEMENT_KIND_UNSUPPORTED = 1; + RESOURCE_ENFORCEMENT_KIND_KERNEL_LIMIT = 2; + // Admission by the platform scheduler, not dedicated physical capacity. + RESOURCE_ENFORCEMENT_KIND_SCHEDULER_REQUEST = 3; +} + +enum ResourceEnforcementScope { + RESOURCE_ENFORCEMENT_SCOPE_UNSPECIFIED = 0; + RESOURCE_ENFORCEMENT_SCOPE_PROCESS = 1; + RESOURCE_ENFORCEMENT_SCOPE_CONTAINER = 2; + RESOURCE_ENFORCEMENT_SCOPE_SANDBOX_ACCOUNT = 3; + RESOURCE_ENFORCEMENT_SCOPE_VIRTUAL_MACHINE = 4; +} + +message SampledResourceBudgetCapabilities { + bool cpu_time_supported = 1; + bool resident_bytes_supported = 2; + ResourceEnforcementScope scope = 3; + // Sleep between completed observations; collection adds latency. This is + // neither an enforcement deadline nor a bound on overshoot. Short-lived + // processes may escape observation. + google.protobuf.Duration sampling_interval = 4; + bool terminate_on_excess = 5; + // Smallest exactly representable CPU-time request. + google.protobuf.Duration cpu_time_resolution = 6; + // A per-process CPU-time kernel limit also accompanies the sampled budget. + // It may use a coarser resolution than the account-wide observation. + bool per_process_cpu_limit = 7; +} diff --git a/python/openshell/sandbox.py b/python/openshell/sandbox.py index c02a33cdbf..89c63e290e 100644 --- a/python/openshell/sandbox.py +++ b/python/openshell/sandbox.py @@ -28,6 +28,7 @@ datamodel_pb2, openshell_pb2, openshell_pb2_grpc, + sandbox_pb2, ) from .errors import _error_mapping_channel, from_grpc_error from .mutations import DeletionOutcome, DeletionResult @@ -147,6 +148,16 @@ def _retry_unrouted(error: grpc.RpcError) -> bool: ) +def _require_sampled_budget_support(stub: Any, timeout: float) -> None: + # This protocol acknowledgement is public; driver guarantees are checked + # by the gateway. Never write fields an older gateway would discard. + response = stub.Health(openshell_pb2.HealthRequest(), timeout=timeout) + if not response.supports_sampled_resource_budget: + raise SandboxError( + "gateway does not acknowledge sampled resource budgets; upgrade it before creating this workload" + ) + + def _workspace_scope(workspace: str) -> datamodel_pb2.WorkspaceSelector: if not workspace: raise ValueError("workspace must be non-empty") @@ -925,6 +936,8 @@ def create( service_exposures: Sequence[ServiceExposure] | None = None, ) -> SandboxRef: request_spec = spec if spec is not None else _default_spec() + if request_spec.resource_requirements.HasField("sampled_budget"): + _require_sampled_budget_support(self._stub, self._timeout) response = self._stub.CreateSandbox( openshell_pb2.CreateSandboxRequest( workspace_scope=_workspace_scope(workspace), @@ -1497,6 +1510,7 @@ def create( memory: str | None = None, gpu_count: int | None = None, gpu: bool = False, + sampled_budget: sandbox_pb2.SampledResourceBudget | None = None, driver_config: Mapping[str, Any] | None = None, ) -> openshell_pb2.SandboxWorkloadTemplate: if template is None: @@ -1512,10 +1526,14 @@ def create( memory=memory, gpu_count=gpu_count, gpu=gpu, + sampled_budget=sampled_budget, driver_config=driver_config, ) elif ( - any(value is not None for value in (name, image, cpu, memory, gpu_count)) + any( + value is not None + for value in (name, image, cpu, memory, gpu_count, sampled_budget) + ) or any( bool(value) for value in (labels, annotations, environment, driver_config) @@ -1526,6 +1544,9 @@ def create( "template cannot be combined with template builder fields" ) + if template.spec.workload.resources.HasField("sampled_budget"): + _require_sampled_budget_support(self._stub, self._timeout) + response = self._stub.CreateSandboxTemplate( openshell_pb2.CreateSandboxTemplateRequest( workspace_scope=_workspace_scope(workspace), @@ -2040,6 +2061,7 @@ def _sandbox_workload_template( memory: str | None = None, gpu_count: int | None = None, gpu: bool = False, + sampled_budget: sandbox_pb2.SampledResourceBudget | None = None, driver_config: Mapping[str, Any] | None = None, ) -> openshell_pb2.SandboxWorkloadTemplate: if gpu_count is not None and gpu_count <= 0: @@ -2059,6 +2081,8 @@ def _sandbox_workload_template( template.spec.workload.image = image if environment: template.spec.workload.environment.update(dict(environment)) + if sampled_budget is not None: + template.spec.workload.resources.sampled_budget.CopyFrom(sampled_budget) if cpu is not None: template.spec.workload.resources.cpu = cpu if memory is not None: diff --git a/python/openshell/sandbox_test.py b/python/openshell/sandbox_test.py index 3fed31734e..255f1ec334 100644 --- a/python/openshell/sandbox_test.py +++ b/python/openshell/sandbox_test.py @@ -21,7 +21,7 @@ from google.protobuf.struct_pb2 import Struct import openshell.sandbox as sandbox_module -from openshell._proto import openshell_pb2, openshell_pb2_grpc +from openshell._proto import openshell_pb2, openshell_pb2_grpc, sandbox_pb2 from openshell.mutations import DeletionOutcome from openshell.sandbox import ( _OIDC_TOKEN_EXPIRY_GRACE_SECONDS, @@ -2110,11 +2110,20 @@ def __init__( self.delete_template_request: ( openshell_pb2.DeleteSandboxTemplateRequest | None ) = None + self.sampled_budget_ack = False self._listed = listed or [] self._listed_pages = listed_pages self.list_requests: list[openshell_pb2.ListSandboxesRequest] = [] self._templates: list[openshell_pb2.SandboxWorkloadTemplate] = [] + def Health( + self, request: object, timeout: float | None = None + ) -> openshell_pb2.HealthResponse: + _ = request, timeout + return openshell_pb2.HealthResponse( + supports_sampled_resource_budget=self.sampled_budget_ack + ) + def GetSandbox( self, request: openshell_pb2.GetSandboxRequest, @@ -2426,6 +2435,7 @@ def test_create_from_template_rejects_empty_workload_template() -> None: def test_sandbox_template_create_builds_template_from_public_fields() -> None: stub = _FakeSandboxStub() + stub.sampled_budget_ack = True client = _template_client_with_fake_stub(stub) created = client.create( @@ -2438,6 +2448,7 @@ def test_sandbox_template_create_builds_template_from_public_fields() -> None: cpu="1", memory="512Mi", gpu_count=2, + sampled_budget=sandbox_pb2.SampledResourceBudget(resident_bytes=4096), driver_config={"kubernetes": {"runtime_class_name": "kata"}}, ) @@ -2457,6 +2468,40 @@ def test_sandbox_template_create_builds_template_from_public_fields() -> None: assert isinstance(config, Struct) assert config["runtime_class_name"] == "kata" + assert template.spec.workload.resources.sampled_budget.resident_bytes == 4096 + + +@pytest.mark.parametrize("template", [False, True]) +def test_sampled_budget_requires_gateway_acknowledgement(template: bool) -> None: + stub = _FakeSandboxStub() + budget = sandbox_pb2.SampledResourceBudget(resident_bytes=4096) + if template: + template_client = _template_client_with_fake_stub(stub) + + def write() -> object: + return template_client.create( + workspace="default", name="budget", sampled_budget=budget + ) + else: + client = _client_with_fake_stub(stub) + spec = openshell_pb2.SandboxSpec() + spec.resource_requirements.sampled_budget.CopyFrom(budget) + + def write() -> object: + return client.create(workspace="default", name="budget", spec=spec) + + with pytest.raises(SandboxError, match="does not acknowledge"): + write() + assert stub.create_request is None + assert stub.create_template_request is None + stub.sampled_budget_ack = True + write() + assert ( + (stub.create_template_request is not None) + if template + else (stub.create_request is not None) + ) + def test_sandbox_template_create_materializes_default_workload() -> None: stub = _FakeSandboxStub() diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 8b64c9768d..567b9c96c5 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -348,3 +348,7 @@ mise run e2e:sdk:ts:exec # public interactive helper against an isolated Dock ``` Formatting and linting are handled by [Biome](https://biomejs.dev) (`biome.json`): 2-space indent, single quotes, semicolons, 120-column width. Generated `src/gen/` is excluded. `sdk:ts:lint` runs in CI as part of `sdk:ts:ci`. + +## Next minor raw message migration + +Raw Health responses add `supportsSampledResourceBudget`. Complete object literals need the false default, or use a generated message builder. Resource requirement/capability messages also gain optional enforcement and budget fields. This change belongs to the next minor release; it is not a patch-source-compatibility claim. Upgrade gateway writers together so older protobuf writers cannot discard the new metadata. diff --git a/sdk/typescript/src/client.test.ts b/sdk/typescript/src/client.test.ts index 86fdfa3972..682351285b 100644 --- a/sdk/typescript/src/client.test.ts +++ b/sdk/typescript/src/client.test.ts @@ -761,22 +761,39 @@ describe('create', () => { }); it('sends canonical main process fields', async () => { - let created: { spec?: { command?: string[]; tty?: boolean } } = {}; + let created: { + spec?: { + command?: string[]; + tty?: boolean; + resourceRequirements?: { + sampledBudget?: { residentBytes?: bigint; cpuTime?: { seconds?: bigint; nanos?: number } }; + }; + }; + } = {}; + let supportsSampledResourceBudget = false; const sandbox = client({ + health: () => ({ supportsSampledResourceBudget }), createSandbox: (req) => { created = req; return readySandbox('sb', 'sb-id'); }, }); - await sandbox.create({ + const spec = { image: 'img', command: ['/opt/worker', '--serve'], tty: true, - }); + sampledBudget: { cpuTime: { seconds: 2n, nanos: 0 }, residentBytes: 4096n }, + }; + await expect(sandbox.create(spec)).rejects.toMatchObject({ code: 'invalid_config' }); + expect(created).toEqual({}); + supportsSampledResourceBudget = true; + await sandbox.create(spec); expect(created.spec?.command).toEqual(['/opt/worker', '--serve']); expect(created.spec?.tty).toBe(true); + expect(created.spec?.resourceRequirements?.sampledBudget?.residentBytes).toBe(4096n); + expect(created.spec?.resourceRequirements?.sampledBudget?.cpuTime?.seconds).toBe(2n); }); it('sends the restart policy', async () => { @@ -1165,6 +1182,37 @@ describe('create', () => { }); }); +describe('sampled budget compatibility', () => { + it('blocks template and raw-spec writes until the gateway acknowledges the fields', async () => { + let supported = false; + let writes = 0; + const transport = createRouterTransport((router) => { + router.service(OpenShell, { + health: () => ({ supportsSampledResourceBudget: supported }), + createSandboxTemplate: (req) => { + writes++; + return { template: req.template }; + }, + createSandbox: () => { + writes++; + return readySandbox('budget', 'sb-id'); + }, + }); + }); + const templates = new SandboxTemplateClient(transport); + const sandbox = new SandboxClient(transport); + const template = { spec: { workload: { resources: { sampledBudget: { residentBytes: 4096n } } } } }; + const spec = { rawSpec: { resourceRequirements: { sampledBudget: { residentBytes: 4096n } } } }; + await expect(templates.create(template)).rejects.toMatchObject({ code: 'invalid_config' }); + await expect(sandbox.create(spec)).rejects.toMatchObject({ code: 'invalid_config' }); + expect(writes).toBe(0); + supported = true; + await templates.create(template); + await sandbox.create(spec); + expect(writes).toBe(2); + }); +}); + describe('sandbox templates', () => { it('create sends the template resource and workspace', async () => { let observed: ScopedRequest & { diff --git a/sdk/typescript/src/client.ts b/sdk/typescript/src/client.ts index 205796f1bd..ee76fa0b47 100644 --- a/sdk/typescript/src/client.ts +++ b/sdk/typescript/src/client.ts @@ -38,7 +38,13 @@ import { type TcpForwardFrameSchema, } from './gen/openshell_pb.js'; import type { EffectiveSetting, GetSandboxConfigResponse, SandboxPolicy, SettingValue } from './gen/sandbox_pb.js'; -import { PolicySource, type SandboxPolicySchema, SettingScope, type SettingValueSchema } from './gen/sandbox_pb.js'; +import { + PolicySource, + type SampledResourceBudgetSchema, + type SandboxPolicySchema, + SettingScope, + type SettingValueSchema, +} from './gen/sandbox_pb.js'; import { ProviderClient } from './providers.js'; import { heldUntilReady, OWNER_REPLICA_HEADER, OWNER_REQUEST_HEADER, routedStream } from './routing.js'; import { validateSshResponse } from './ssh-validate.js'; @@ -80,10 +86,22 @@ export type { SandboxWorkloadTemplate, SandboxWorkloadTemplateSpec, } from './gen/openshell_pb.js'; -export type { SandboxPolicy, SettingValue } from './gen/sandbox_pb.js'; +export type { SampledResourceBudget, SandboxPolicy, SettingValue } from './gen/sandbox_pb.js'; export type { ConnectOptions }; export { errorCode }; +// Health acknowledges that the gateway preserves sampled fields on writes. +// It is available to workspace users; driver admission still decides support. +async function requireSampledBudgetSupport(grpc: Client): Promise { + const health = await grpc.health({}); + if (!health.supportsSampledResourceBudget) { + throw new SdkError( + 'invalid_config', + 'gateway does not acknowledge sampled resource budgets; upgrade it before creating this workload', + ); + } +} + // ---- Curated public types -------------------------------------------------- // The gateway enums (SandboxPhase, ServiceStatus, SettingScope, PolicySource) @@ -164,6 +182,8 @@ export interface SandboxSpec { environment?: Record; providers?: string[]; gpu?: boolean; + /** Explicit sampled budgets, distinct from hard CPU and memory quotas. */ + sampledBudget?: MessageInitShape; /** Exact canonical command. Empty selects the gateway scratch shell. */ command?: string[]; /** Allocate a retained pseudo-terminal for the canonical command. */ @@ -1118,6 +1138,9 @@ export class SandboxTemplateClient { options?: SandboxTemplateWorkspaceOptions | null, ): Promise { try { + if (template.spec?.workload?.resources?.sampledBudget !== undefined) { + await requireSampledBudgetSupport(this.grpc); + } const resp = await this.grpc.createSandboxTemplate({ workspaceScope: workspaceScope(options), template, @@ -1221,7 +1244,10 @@ export class SandboxClient { environment: spec.environment ?? {}, providers: spec.providers ?? [], template: spec.image ? { image: spec.image } : undefined, - resourceRequirements: spec.gpu ? { gpu: {} } : undefined, + resourceRequirements: + spec.gpu || spec.sampledBudget !== undefined + ? { gpu: spec.gpu ? {} : undefined, sampledBudget: spec.sampledBudget } + : undefined, policy: spec.policy, command: spec.command ?? [], tty: spec.tty ?? false, @@ -1229,6 +1255,9 @@ export class SandboxClient { }; if (spec.rawSpec) Object.assign(specInit, spec.rawSpec); + if (specInit.resourceRequirements?.sampledBudget !== undefined) { + await requireSampledBudgetSupport(this.grpc); + } const resp = await this.grpc.createSandbox({ workspaceScope: workspaceScope(spec), name: spec.name ?? '', @@ -1243,7 +1272,7 @@ export class SandboxClient { }); return sandboxRef(resp.sandbox, resp.serviceUrls); } catch (e) { - throw fromConnect(e); + throw e instanceof SdkError ? e : fromConnect(e); } } diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index 4948c13623..b041f3d320 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -29,6 +29,7 @@ export type { ProviderChange, ProviderChangeOptions, ProviderRef, + SampledResourceBudget, SandboxConfig, SandboxFromTemplateSpec, SandboxPhaseName, diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index b757172b7a..7f8e48cc10 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -367,6 +367,15 @@ config opt-in. Consult the published gateway configuration reference before changing admission settings; do not recommend disabling admission to bypass a denial. Put driver config on a template only when it should be reused. +For a compute driver that advertises sampled account budgets, use +`sandbox template create` with `--cpu-time-budget 30s` and +`--resident-memory-budget-bytes 1073741824`, then create from that template. +These explicit budgets terminate on observed excess; collection and sampling +can overshoot. They are distinct from `--cpu` core-rate limits and `--memory` +hard limits. Inspect `gateway info` for driver guarantees; ordinary workspace +users do not need its administrator permission to use budgets. Budget requests +fail before provisioning if the gateway or driver does not support them. + ### Manage sandbox workload templates ```bash