feat(sandbox): support storage-preserving sandbox configuration updates

.agents/skills/helm-dev-environment/SKILL.md

@@ -421,6 +421,30 @@ above) instead of relying only on `OPENSHELL_GATEWAY_ENDPOINT`; some tests call
 ### Kubernetes E2E Notes
 
 Use `mise run e2e:kubernetes` for the standard Helm-backed Kubernetes suite.
+Set `OPENSHELL_E2E_KUBE_TEST=kubernetes_image_update` to scope the Rust suite to
+combined image, command, environment, and metadata updates with retained sandbox,
+SSH, and PVC identities. The next start verifies the new argv and environment. This target is
+included in both existing Agent Sandbox API-version lanes; unlike the Python
+Docker suite, it exercises the driver that supports image updates.
+
+Image updates default to disabled outside test fixtures. The e2e wrapper opts
+in only for a fresh gateway fixture that it owns; set
+`OPENSHELL_E2E_KUBE_IMAGE_UPDATES=0` to prevent that opt-in. Do not propagate
+activation to an existing shared gateway or enable it during a mixed-version
+rollout. For manual testing, follow the retirement and activation checklist in
+`docs/kubernetes/high-availability.mdx` before setting
+`gatewayConfig: {"openshell.gateway": {enable_sandbox_updates: true}}`.
+The gateway process equivalents are `--enable-sandbox-updates true` and
+`OPENSHELL_ENABLE_SANDBOX_UPDATES=true`. Retire every old process and Job,
+settle outstanding driver work, prevent old binaries returning, and do not
+treat PostgreSQL connection markers or membership TTL expiry as retirement
+proof. SQLite testing supports one gateway process only. Image-update clients
+must call the separate `SandboxUpdates` service on the existing gateway
+listener, not add a method to the baseline `OpenShell` service or inspect new
+public `SandboxStatus` image fields. Resume a pending operation with
+`SandboxUpdates.UpdateSandbox`, the current resource version, and no selected
+fields, following the documented request-ID semantics.
+
 The kube e2e wrapper creates only one port-forward, to `svc/openshell`; it no
 longer forwards the unauthenticated health listener or runs a `/readyz` e2e
 target. `/readyz` remains covered by server unit/integration tests.
@@ -611,3 +635,13 @@ for dependencies still declared in `Chart.yaml`.
 | `deploy/kube/manifests/envoy-gateway-openshell.yaml` | GatewayClass and BackendTrafficPolicy for Envoy Gateway (`mise run helm:gateway:apply`) |
 | `tasks/scripts/helm-k3s-local.sh` | k3d cluster create/delete/start/stop/status |
 | `tasks/scripts/keycloak-k8s-setup.sh` | Keycloak deploy, realm import, and development TLS trust anchor |
+
+## Sandbox update regression lane
+
+Run `mise run e2e:kubernetes:updates` for the testkit update suite against both
+supported Agent Sandbox APIs with the combined Helm chart and its bundled
+workspace resources. It verifies stopped-only updates, retained CR/PVC
+identity and contents, command/environment changes, and request replay.
+`OPENSHELL_E2E_UPDATE_ORIGINAL_IMAGE` and `OPENSHELL_E2E_UPDATE_TARGET_IMAGE` can
+select mirrored Alpine 3.21 and 3.22 manifests. Defaults use the public registry
+cache; keep immutable digests when overriding them.

crates/openshell-cli/src/run.rs

@@ -4116,6 +4116,14 @@ pub async fn sandbox_stop(
         .into_inner()
         .sandbox
         .ok_or_else(|| miette!("gateway returned no sandbox after stop"))?;
+    // A successful stop can preserve a canonical result when the transient
+    // compute resource is already gone. There is no Stopped event to wait for.
+    if sandbox.phase() == SandboxPhase::Completed as i32
+        || (sandbox.phase() == SandboxPhase::Error as i32 && has_main_process_result(&sandbox))
+    {
+        println!("{} Sandbox {name} already finished", "✓".green().bold());
+        return Ok(());
+    }
     wait_for_lifecycle_phase(&mut client, sandbox, SandboxPhase::Stopped).await?;
     println!("{} Stopped sandbox {name}", "✓".green().bold());
     Ok(())

crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs

@@ -67,6 +67,7 @@ struct SandboxState {
     create_requests: Arc<Mutex<Vec<CreateSandboxRequest>>>,
     expose_service_requests: Arc<Mutex<Vec<openshell_core::proto::ExposeServiceRequest>>>,
     fail_delete_sandbox_message: Arc<Mutex<Option<String>>>,
+    stop_response: Arc<Mutex<Option<Sandbox>>>,
     vm_error_after_started: Arc<AtomicBool>,
     vm_error_with_observed_exit: Arc<AtomicBool>,
     vm_slow_progress_before_ready: Arc<AtomicBool>,
@@ -231,7 +232,17 @@ impl OpenShell for TestOpenShell {
         &self,
         _request: tonic::Request<openshell_core::proto::StopSandboxRequest>,
     ) -> Result<Response<SandboxResponse>, Status> {
-        Err(Status::unimplemented("unused"))
+        let sandbox = self
+            .state
+            .stop_response
+            .lock()
+            .await
+            .clone()
+            .ok_or_else(|| Status::unimplemented("unused"))?;
+        Ok(Response::new(SandboxResponse {
+            sandbox: Some(sandbox),
+            ..Default::default()
+        }))
     }
 
     async fn start_sandbox(
@@ -1709,6 +1720,57 @@ fn test_config() -> run::SandboxCreateConfig<'static> {
     }
 }
 
+#[tokio::test]
+async fn sandbox_stop_accepts_terminal_results_without_waiting_for_stopped() {
+    let server = run_server().await;
+    let tls = test_tls(&server);
+    let config_dir = helpers::tempdir().unwrap();
+    for (phase, exit_code, reason, succeeds) in [
+        (
+            SandboxPhase::Completed,
+            Some(0),
+            "MainProcessCompleted",
+            true,
+        ),
+        (SandboxPhase::Error, Some(17), "MainProcessFailed", true),
+        (SandboxPhase::Stopped, None, "Stopped", true),
+        (
+            SandboxPhase::Error,
+            Some(17),
+            "ComputeResourceMissing",
+            false,
+        ),
+    ] {
+        let sandbox = Sandbox {
+            status: Some(SandboxStatus {
+                phase: phase as i32,
+                exit_code,
+                conditions: vec![SandboxCondition {
+                    r#type: "Ready".into(),
+                    status: "False".into(),
+                    reason: reason.into(),
+                    ..Default::default()
+                }],
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        *server.openshell.state.stop_response.lock().await = Some(sandbox);
+        Environment::new()
+            .set("XDG_CONFIG_HOME", config_dir.path())
+            .run_async(async {
+                let result = tokio::time::timeout(
+                    Duration::from_secs(5),
+                    run::sandbox_stop(&server.endpoint, "terminal-stop-test", "default", &tls),
+                )
+                .await
+                .expect("stop must finish without watching for another phase");
+                assert_eq!(result.is_ok(), succeeds, "{reason}: {result:?}");
+            })
+            .await;
+    }
+}
+
 #[tokio::test]
 async fn sandbox_delete_continues_after_entry_failure() {
     let server = run_server().await;

crates/openshell-core/README.md

@@ -65,6 +65,20 @@ configuration validation live in `openshell_core::middleware`. Policy admission
 and the supervisor runtime consume the same contract without introducing a
 dependency from the policy crate to the supervisor implementation.
 
+## Optional Compute Updates
+
+The generated `proto::compute::v1::compute_driver_server::ComputeDriver` trait
+retains its baseline methods and streaming associated type. Supporting external
+drivers additionally serve `ComputeDriverConfigurationUpdates` on the same
+endpoint and advertise `openshell.compute.update-sandbox-configuration`.
+The request carries image and lifecycle preconditions plus optional command and
+environment replacements. Drivers without the capability need no new method.
+
+Linked drivers opt in through `openshell_core::compute::ComputeDriver` and
+`ComputeDriverFactory::build_with_configuration_updates`. The default factory
+hook calls `build` without an updater. Registration and capability negotiation
+are both required. Metadata-only updates are handled by the gateway.
+
 ## Protobuf Struct Conversion
 
 Use `openshell_core::proto_struct` when crossing between `serde_json` values and

crates/openshell-core/src/compute.rs

@@ -0,0 +1,27 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Opt-in in-process sandbox updates, separate from the baseline driver contract.
+
+use crate::proto::compute::v1::compute_driver_server;
+
+/// Optional in-process configuration updater, registered separately by a driver factory.
+///
+/// Existing drivers need only the generated baseline trait, not this extension.
+/// Supporting factories opt in through `ComputeDriverFactory::build_with_configuration_updates`
+/// and advertise the configuration-update capability. External servers use the
+/// separate generated `ComputeDriverConfigurationUpdates` service on the same endpoint.
+#[tonic::async_trait]
+pub trait ComputeDriver: compute_driver_server::ComputeDriver {
+    async fn update_sandbox_configuration(
+        &self,
+        _request: tonic::Request<crate::proto::compute::v1::UpdateSandboxConfigurationRequest>,
+    ) -> Result<
+        tonic::Response<crate::proto::compute::v1::UpdateSandboxConfigurationResponse>,
+        tonic::Status,
+    > {
+        Err(tonic::Status::unimplemented(
+            "sandbox configuration updates are not supported by this driver",
+        ))
+    }
+}

crates/openshell-core/src/config.rs

@@ -252,6 +252,11 @@ pub struct Config {
     /// Disabled for local gateways by default.
     pub enable_websocket_tunnel: bool,
 
+    /// Operator activation after all gateways and outstanding legacy driver
+    /// operations have been drained onto the configuration-update-compatible protocol.
+    /// Disabled by default; not a license to run mixed-version writers.
+    pub enable_sandbox_updates: bool,
+
     /// Disabled-by-default gateway interceptor service configs.
     pub gateway_interceptors: Vec<GatewayInterceptorConfig>,
 
@@ -999,6 +1004,7 @@ impl Config {
             oidc: None,
             auth: GatewayAuthConfig::default(),
             enable_websocket_tunnel: false,
+            enable_sandbox_updates: false,
             gateway_interceptors: Vec::new(),
             provider_profile_sources: vec![GatewayProviderProfileSourceConfig::User],
             mtls_auth: MtlsAuthConfig::default(),

crates/openshell-core/src/extension_protocol.rs

@@ -25,6 +25,10 @@ pub const COMPUTE_ISOLATION_BACKEND: &str = "openshell.compute.isolation-backend
 /// Driver implements `SuspendSandbox` and resuming through `StartSandbox`.
 pub const COMPUTE_SUSPEND_RESUME: &str = "openshell.compute.suspend-resume";
 
+/// Optional stopped-state workload configuration updates retaining the resource and storage.
+pub const COMPUTE_UPDATE_SANDBOX_CONFIGURATION: &str =
+    "openshell.compute.update-sandbox-configuration";
+
 const MAX_IMPLEMENTATION_NAME_BYTES: usize = 128;
 const MAX_IMPLEMENTATION_VERSION_BYTES: usize = 128;
 const MAX_CAPABILITY_BYTES: usize = 128;
@@ -128,6 +132,7 @@ pub fn gateway_metadata(family: ExtensionFamily) -> PeerMetadata {
     if family == ExtensionFamily::Compute {
         supported_capabilities.push(COMPUTE_LAUNCH_AUTHENTICATION.to_string());
         supported_capabilities.push(COMPUTE_ISOLATION_BACKEND.to_string());
+        supported_capabilities.push(COMPUTE_UPDATE_SANDBOX_CONFIGURATION.to_string());
     }
     PeerMetadata {
         protocol_version: Some(ProtocolVersion {
@@ -399,6 +404,23 @@ fn valid_capability(value: &str) -> bool {
 mod tests {
     use super::*;
 
+    #[test]
+    fn compute_gateway_advertises_both_optional_extensions() {
+        let gateway = gateway_metadata(ExtensionFamily::Compute);
+        assert!(
+            gateway
+                .supported_capabilities
+                .iter()
+                .any(|c| c == COMPUTE_ISOLATION_BACKEND)
+        );
+        assert!(
+            gateway
+                .supported_capabilities
+                .iter()
+                .any(|c| c == COMPUTE_UPDATE_SANDBOX_CONFIGURATION)
+        );
+    }
+
     fn compatible() -> (PeerMetadata, PeerMetadata) {
         let gateway = gateway_metadata(ExtensionFamily::Compute);
         let extension = extension_metadata(

crates/openshell-core/src/lib.rs

@@ -11,6 +11,7 @@
 
 pub mod activity;
 pub mod auth;
+pub mod compute;
 pub mod config;
 pub mod container_paths;
 pub mod denial;
@@ -58,6 +59,7 @@ pub mod rpc_error;
 pub mod sandbox_env;
 pub mod sandbox_generation;
 pub mod sandbox_session;
+pub mod sandbox_update;
 pub mod secrets;
 pub mod settings;
 pub mod shell;

crates/openshell-core/src/sandbox_update.rs

@@ -0,0 +1,76 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Shared projections for selected stopped-sandbox configuration changes.
+use crate::proto::{Sandbox, SandboxCommandUpdate, SandboxMapUpdate};
+use std::collections::HashMap;
+
+/// Gateway-owned identity and lifecycle metadata is never caller-replaceable.
+/// Other keys, including user keys under `OpenShell` domains, remain editable.
+#[must_use]
+pub fn is_system_metadata_key(key: &str) -> bool {
+    key.starts_with("internal.openshell.ai/")
+        || matches!(
+            key,
+            "openshell.nvidia.com/retention"
+                | "openshell.ai/managed-by"
+                | "openshell.ai/sandbox-id"
+                | "openshell.ai/sandbox-name"
+                | "openshell.ai/sandbox-namespace"
+                | "openshell.ai/sandbox-workspace"
+                | "openshell.ai/gateway-id"
+                | "openshell.ai/caller-driver-config-used"
+                | "openshell.ai/resource-admission-identities"
+        )
+}
+
+fn replace_user_metadata(current: &mut HashMap<String, String>, update: &SandboxMapUpdate) {
+    current.retain(|key, _| is_system_metadata_key(key));
+    current.extend(update.values.clone());
+}
+
+/// Caller must validate syntax and forbid system keys before projecting.
+pub fn apply_configuration(
+    sandbox: &mut Sandbox,
+    command: Option<&SandboxCommandUpdate>,
+    environment: Option<&SandboxMapUpdate>,
+    labels: Option<&SandboxMapUpdate>,
+    annotations: Option<&SandboxMapUpdate>,
+) {
+    if let Some(spec) = sandbox.spec.as_mut() {
+        if let Some(command) = command {
+            spec.command.clone_from(&command.argv);
+        }
+        if let Some(environment) = environment {
+            spec.environment.clone_from(&environment.values);
+        }
+    }
+    if let Some(metadata) = sandbox.metadata.as_mut() {
+        if let Some(labels) = labels {
+            replace_user_metadata(&mut metadata.labels, labels);
+        }
+        if let Some(annotations) = annotations {
+            replace_user_metadata(&mut metadata.annotations, annotations);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn replacement_preserves_owned_keys_without_freezing_user_domains() {
+        let mut metadata = HashMap::from([
+            ("internal.openshell.ai/auth-epoch".into(), "trusted".into()),
+            ("openshell.nvidia.com/retention".into(), "ephemeral".into()),
+            ("openshell.ai/user-note".into(), "old".into()),
+            ("agents.x-k8s.io/user-note".into(), "old".into()),
+        ]);
+        replace_user_metadata(&mut metadata, &SandboxMapUpdate::default());
+        assert_eq!(metadata.len(), 2);
+        assert_eq!(metadata["openshell.nvidia.com/retention"], "ephemeral");
+        assert!(!is_system_metadata_key("openshell.ai/user-note"));
+        assert!(!is_system_metadata_key("agents.x-k8s.io/user-note"));
+    }
+}