From 24050283ca3d96a683ab709257be7effda886c84 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Mon, 26 Oct 2026 23:23:46 -0700 Subject: [PATCH 1/5] feat(supervisor): add independent application listener and endpoints Signed-off-by: Taylor Mutch --- .agents/skills/helm-dev-environment/SKILL.md | 11 + Cargo.lock | 1 + crates/openshell-core/src/app_listener.rs | 71 + crates/openshell-core/src/config.rs | 2 + crates/openshell-core/src/grpc_client.rs | 3 + crates/openshell-core/src/lib.rs | 1 + .../src/app_listener.rs | 89 + .../openshell-driver-kubernetes/src/config.rs | 45 + .../openshell-driver-kubernetes/src/driver.rs | 79 +- crates/openshell-driver-kubernetes/src/lib.rs | 1 + .../openshell-driver-kubernetes/src/main.rs | 2 + .../src/sandbox_runtime.rs | 171 +- crates/openshell-gateway/src/lib.rs | 1 + crates/openshell-server/src/cli.rs | 9 + crates/openshell-server/src/config_file.rs | 2 + crates/openshell-server/src/grpc/policy.rs | 18 +- crates/openshell-server/src/grpc/service.rs | 171 ++ crates/openshell-server/src/lib.rs | 8 + crates/openshell-server/src/storage_proto.rs | 4 +- crates/openshell-supervisor/Cargo.toml | 4 +- crates/openshell-supervisor/README.md | 33 + .../openshell-supervisor/src/app_listener.rs | 1446 +++++++++++++++++ .../src/backend_setup/tests.rs | 2 + .../src/config_runtime.rs | 56 +- .../src/legacy_startup_regression_tests.rs | 1 + crates/openshell-supervisor/src/lib.rs | 32 + crates/openshell-supervisor/src/main.rs | 7 + .../openshell-supervisor/src/shared_host.rs | 10 + .../openshell/ci/values-app-listener.yaml | 14 + deploy/helm/openshell/templates/_helpers.tpl | 37 + deploy/helm/openshell/templates/_toml.tpl | 3 + .../helm/openshell/templates/supervisor.yaml | 37 + .../openshell/tests/app_listener_test.yaml | 74 + deploy/helm/test-gateway-config-parser.sh | 7 + docs/how-it-works/gateways/configuration.mdx | 41 + e2e/kubernetes/app-listener/README.md | 77 + proto/sandbox.proto | 13 + sdk/go/proto/sandboxv1/sandbox.pb.go | 230 ++- skills/debug-openshell-cluster/SKILL.md | 24 + 39 files changed, 2655 insertions(+), 182 deletions(-) create mode 100644 crates/openshell-core/src/app_listener.rs create mode 100644 crates/openshell-driver-kubernetes/src/app_listener.rs create mode 100644 crates/openshell-supervisor/src/app_listener.rs create mode 100644 deploy/helm/openshell/ci/values-app-listener.yaml create mode 100644 deploy/helm/openshell/tests/app_listener_test.yaml create mode 100644 e2e/kubernetes/app-listener/README.md diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index df8189ad71..d3d4cdfb7f 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -645,3 +645,14 @@ identity and contents, command/environment changes, and request replay. `OPENSHELL_E2E_UPDATE_ORIGINAL_IMAGE` and `OPENSHELL_E2E_UPDATE_TARGET_IMAGE` can select mirrored Alpine 3.21 and 3.22 manifests. Defaults use the public registry cache; keep immutable digests when overriding them. + +## App listener fixture + +The opt-in `deploy/helm/openshell/ci/values-app-listener.yaml` enables the +supervisor listener at port 9443 without discovery or a proxy. Provision the +operator TLS Secret and an ingress-client NetworkPolicy first. Follow +`e2e/kubernetes/app-listener/README.md` for direct in-cluster verification, +shared-member selection, and the HA follow-up checks. Keep the fixture disabled +in default profiles. Shared supervisors expose app ports on their Pod IPs +without a shared app Service. Select the member owning the sandbox; automatic +owner selection belongs to the later discovery API. diff --git a/Cargo.lock b/Cargo.lock index 92b54ed89c..d81d1a54b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5166,6 +5166,7 @@ dependencies = [ name = "openshell-supervisor" version = "0.0.0" dependencies = [ + "axum", "bytes", "clap", "futures", diff --git a/crates/openshell-core/src/app_listener.rs b/crates/openshell-core/src/app_listener.rs new file mode 100644 index 0000000000..6fe2d1e2b4 --- /dev/null +++ b/crates/openshell-core/src/app_listener.rs @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Driver-independent application listener configuration. + +use serde::{Deserialize, Serialize}; + +/// Explicitly enabled application ingress. Deployment details such as TLS +/// Secret names belong to the driver, not to this common configuration. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct AppListenerConfig { + pub enabled: bool, + pub port: u16, + pub domain: String, +} + +impl Default for AppListenerConfig { + fn default() -> Self { + Self { + enabled: false, + port: 8443, + domain: String::new(), + } + } +} + +impl AppListenerConfig { + pub fn validate(&self) -> Result<(), String> { + if !self.enabled { + return Ok(()); + } + if self.port < 1024 { + return Err("supervisor.app_listener.port must be at least 1024".into()); + } + if self.domain.len() > 253 + || !self.domain.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + }) + { + return Err("supervisor.app_listener.domain must be a lowercase DNS name".into()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_explicit_enablement_requires_a_valid_endpoint() { + assert!(!AppListenerConfig::default().enabled); + assert!(AppListenerConfig::default().validate().is_ok()); + let mut config = AppListenerConfig { + enabled: true, + ..Default::default() + }; + assert!(config.validate().is_err()); + config.domain = "services.example.com".into(); + assert!(config.validate().is_ok()); + config.port = 0; + assert!(config.validate().is_err()); + } +} diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 308b9edb61..b88076100b 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -337,6 +337,7 @@ pub struct Config { /// Browser-facing sandbox service routing configuration. pub service_routing: ServiceRoutingConfig, + pub supervisor_app_listener: crate::app_listener::AppListenerConfig, } /// Browser-facing sandbox service routing configuration. @@ -1021,6 +1022,7 @@ impl Config { grpc_rate_limit_requests: None, grpc_rate_limit_window_secs: None, service_routing: ServiceRoutingConfig::default(), + supervisor_app_listener: crate::app_listener::AppListenerConfig::default(), } } diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 209d0bddf4..64268a7866 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -1865,6 +1865,7 @@ pub struct CachedOpenShellClient { /// Settings poll result returned by [`CachedOpenShellClient::poll_settings`]. #[derive(Clone, Debug)] pub struct SettingsPollResult { + pub service_routes: Vec, pub configuration_instance_id: String, pub configuration_admitted: bool, pub configuration_error: String, @@ -1892,6 +1893,7 @@ pub struct SettingsPollResult { fn settings_poll_result(inner: crate::proto::GetSandboxConfigResponse) -> SettingsPollResult { SettingsPollResult { + service_routes: inner.service_routes, configuration_instance_id: inner.configuration_instance_id, configuration_admitted: inner.configuration_admitted, configuration_error: inner.configuration_error, @@ -1921,6 +1923,7 @@ fn settings_poll_result(inner: crate::proto::GetSandboxConfigResponse) -> Settin impl From for SettingsPollResult { fn from(snapshot: crate::proto::SandboxConfigSnapshot) -> Self { settings_poll_result(crate::proto::GetSandboxConfigResponse { + service_routes: snapshot.service_routes, policy: snapshot.policy, version: snapshot.version, policy_hash: snapshot.policy_hash, diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index f16eb7c130..83faf18ef0 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -10,6 +10,7 @@ //! - Build version metadata pub mod activity; +pub mod app_listener; pub mod auth; pub mod compute; pub mod config; diff --git a/crates/openshell-driver-kubernetes/src/app_listener.rs b/crates/openshell-driver-kubernetes/src/app_listener.rs new file mode 100644 index 0000000000..0f941d1e04 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/app_listener.rs @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reachable supervisor app endpoints. External ingress routes are owned by +//! ingress integrations, not by this compute driver. + +use crate::config::KubernetesAppListenerConfig; +use k8s_openapi::api::core::v1::{ + ContainerPort, EnvVar, Pod, SecretVolumeSource, Volume, VolumeMount, +}; + +const TLS_PATH: &str = "/etc/openshell/app-tls"; + +pub fn configure_supervisor_pod(mut pod: Pod, config: &KubernetesAppListenerConfig) -> Pod { + if !config.common.enabled { + return pod; + } + let spec = pod.spec.as_mut().expect("supervisor Pod spec"); + let container = spec + .containers + .iter_mut() + .find(|c| c.name == "supervisor") + .expect("supervisor container"); + let env = container.env.get_or_insert_default(); + for (name, value) in [ + ("OPENSHELL_APP_LISTENER_ENABLED", "true".to_string()), + ( + "OPENSHELL_APP_LISTEN_ADDRESS", + format!("0.0.0.0:{}", config.common.port), + ), + ("OPENSHELL_APP_SERVICE_DOMAIN", config.common.domain.clone()), + ("OPENSHELL_APP_TLS_CERT", format!("{TLS_PATH}/tls.crt")), + ("OPENSHELL_APP_TLS_KEY", format!("{TLS_PATH}/tls.key")), + ] { + env.push(EnvVar { + name: name.into(), + value: Some(value), + ..Default::default() + }); + } + container.ports.get_or_insert_default().push(ContainerPort { + name: Some("app".into()), + container_port: i32::from(config.common.port), + protocol: Some("TCP".into()), + ..Default::default() + }); + container + .volume_mounts + .get_or_insert_default() + .push(VolumeMount { + name: "app-tls".into(), + mount_path: TLS_PATH.into(), + read_only: Some(true), + ..Default::default() + }); + spec.volumes.get_or_insert_default().push(Volume { + name: "app-tls".into(), + secret: Some(SecretVolumeSource { + secret_name: Some(config.tls_secret_name.clone()), + ..Default::default() + }), + ..Default::default() + }); + pod +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn disabled_listener_does_not_add_ports_or_private_key_mounts() { + let pod = Pod { + spec: Some(k8s_openapi::api::core::v1::PodSpec { + containers: vec![k8s_openapi::api::core::v1::Container { + name: "supervisor".into(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + let config = KubernetesAppListenerConfig { + tls_secret_name: "app-tls".into(), + ..Default::default() + }; + assert_eq!(configure_supervisor_pod(pod.clone(), &config), pod); + } +} diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index a983459477..0a22fada9e 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -118,6 +118,7 @@ impl FromStr for KubernetesControlConnection { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesSandboxRuntimeConfig { + pub app_listener: KubernetesAppListenerConfig, /// TCP port of the Sandbox Protocol listener: the workload boundary for /// supervisor-initiated sandboxes, the supervisor for runtime-initiated ones. pub boundary_port: u16, @@ -127,12 +128,32 @@ impl Default for KubernetesSandboxRuntimeConfig { fn default() -> Self { Self { boundary_port: 5500, + app_listener: KubernetesAppListenerConfig::default(), } } } +/// Driver-owned TLS deployment inputs. Common listener settings +/// are injected by the gateway so there is only one port/domain source. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct KubernetesAppListenerConfig { + pub tls_secret_name: String, + #[serde(skip)] + pub common: openshell_core::app_listener::AppListenerConfig, +} + impl KubernetesSandboxRuntimeConfig { pub fn validate(&self) -> Result<(), String> { + self.app_listener.common.validate()?; + if self.app_listener.common.enabled { + if !is_dns_subdomain(&self.app_listener.tls_secret_name) { + return Err("sandbox_runtime.app_listener.tls_secret_name is required when the app listener is enabled".into()); + } + if [self.boundary_port, 5501].contains(&self.app_listener.common.port) { + return Err("supervisor.app_listener.port conflicts with a supervisor runtime or health port".into()); + } + } if self.boundary_port < 1024 { return Err("sandbox_runtime.boundary_port must be at least 1024".to_string()); } @@ -557,6 +578,11 @@ impl KubernetesComputeConfig { "supervisor_topology=gateway requires shared_supervisor".to_string() })?; shared.validate()?; + if self.sandbox_runtime.app_listener.common.enabled + && self.sandbox_runtime.app_listener.common.port == shared.runtime_port + { + return Err("supervisor.app_listener.port conflicts with shared_supervisor.runtime_port".into()); + } if self.provider_spiffe_enabled() { return Err("supervisor_topology=gateway does not support provider_spiffe_workload_api_socket_path; a shared supervisor has no per-sandbox workload identity".into()); } @@ -1118,6 +1144,25 @@ mod tests { use super::*; use std::collections::BTreeMap as HashMap; + #[test] + fn app_endpoint_configuration_has_one_port_source_and_explicit_enablement() { + let mut config = KubernetesSandboxRuntimeConfig::default(); + config.app_listener.tls_secret_name = "app-tls".into(); + assert!(!config.app_listener.common.enabled); + assert!(config.validate().is_ok()); + config.app_listener.common = openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: 9443, + domain: "services.example.com".into(), + }; + assert!(config.validate().is_ok()); + config.app_listener.common.port = config.boundary_port; + assert!(config.validate().is_err()); + config.app_listener.common.port = 9443; + config.app_listener.tls_secret_name.clear(); + assert!(config.validate().is_err()); + } + #[test] fn gateway_supervisor_topology_requires_its_connection_and_refuses_unsupported_inputs() { let shared = || SharedSupervisorConfig { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 5737951d13..5ae29112c8 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -489,7 +489,7 @@ pub enum KubernetesDriverError { } impl KubernetesDriverError { - fn from_kube(err: KubeError) -> Self { + pub(crate) fn from_kube(err: KubeError) -> Self { match err { KubeError::Api(api) if api.code == 409 && api.reason == "AlreadyExists" => { Self::AlreadyExists @@ -3368,44 +3368,47 @@ impl KubernetesComputeDriver { let pods: Api = Api::namespaced(self.client.clone(), namespace); pods.create( &PostParams::default(), - &supervisor_pod( - namespace, - names, - sandbox_id, - sandbox_name, - &self.config.gateway_id, - &self.config.supervisor_image, - self.config.supervisor_image_pull_policy, - &self.config.service_account_name, - agent_uid, - agent_gid, - &self.generation_image_pull_secret_names(names), - &self.config.grpc_endpoint, - self.config.supervisor_client_tls(), - main_process_spec, - log_level, - self.config.effective_sa_token_ttl_secs(), - self.config.https_proxy.as_deref(), - self.config.no_proxy.as_deref(), - self.config - .proxy_auth_secret_name - .as_deref() - .zip(self.config.proxy_auth_secret_key.as_deref()), - self.config.proxy_auth_allow_insecure == Some(true), - self.config.proxy_connect_by_hostname == Some(true), - upstream_proxy_ca_bundle_staged, - self.config.provider_spiffe_enabled().then_some( + &crate::app_listener::configure_supervisor_pod( + supervisor_pod( + namespace, + names, + sandbox_id, + sandbox_name, + &self.config.gateway_id, + &self.config.supervisor_image, + self.config.supervisor_image_pull_policy, + &self.config.service_account_name, + agent_uid, + agent_gid, + &self.generation_image_pull_secret_names(names), + &self.config.grpc_endpoint, + self.config.supervisor_client_tls(), + main_process_spec, + log_level, + self.config.effective_sa_token_ttl_secs(), + self.config.https_proxy.as_deref(), + self.config.no_proxy.as_deref(), self.config - .provider_spiffe_workload_api_socket_path - .as_str(), - ), - network_additional_ca - .as_ref() - .map(|generation| (generation.name.as_str(), generation.digest.as_str())), - owner, - &self.supervisor_otlp_environment(), - ) - .map_err(KubernetesDriverError::Message)?, + .proxy_auth_secret_name + .as_deref() + .zip(self.config.proxy_auth_secret_key.as_deref()), + self.config.proxy_auth_allow_insecure == Some(true), + self.config.proxy_connect_by_hostname == Some(true), + upstream_proxy_ca_bundle_staged, + self.config.provider_spiffe_enabled().then_some( + self.config + .provider_spiffe_workload_api_socket_path + .as_str(), + ), + network_additional_ca + .as_ref() + .map(|generation| (generation.name.as_str(), generation.digest.as_str())), + owner, + &self.supervisor_otlp_environment(), + ) + .map_err(KubernetesDriverError::Message)?, + &self.config.sandbox_runtime.app_listener, + ), ) .await .map_err(KubernetesDriverError::from_kube)? diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 4d21fbff51..f3a1e419ec 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +pub mod app_listener; pub mod config; pub mod driver; pub mod grpc; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 0b2b010949..6fac8c30af 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -293,6 +293,8 @@ async fn main() -> Result<()> { .unwrap_or_else(openshell_core::config::default_supervisor_image), supervisor_image_pull_policy: args.supervisor_image_pull_policy, sandbox_runtime: KubernetesSandboxRuntimeConfig { + app_listener: + openshell_driver_kubernetes::config::KubernetesAppListenerConfig::default(), boundary_port: args.sandbox_runtime_boundary_port, }, https_proxy: args.https_proxy, diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 111b6ba7ec..3ae1d837fc 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -1371,84 +1371,105 @@ mod tests { #[test] fn destination_trust_is_mounted_only_into_the_dedicated_supervisor_pod() { - let names = SandboxRuntimeNames::new("pair"); - let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; - let pod = supervisor_pod( - "sandbox", - &names, - "pair", - "demo", - "gateway", - "supervisor:latest", - None, - "sandbox-sa", - 1000, - 1000, - &[], - "https://gateway:8080", - SupervisorClientTls::Disabled, - "{}", - "info", - 600, - None, - None, - None, - false, - false, - false, - None, - Some(("openshell-additional-ca-gateway-generation", digest)), - owner(), - &[], - ) - .expect("render supervisor Pod with destination trust"); - let spec = pod.spec.expect("Pod spec"); - let container = &spec.containers[0]; - assert!(container.command.as_deref().is_some_and(|command| { - command - .windows(2) - .any(|args| args == ["--additional-ca-bundle", ADDITIONAL_CA_MOUNT_PATH]) - })); - assert_eq!( - container - .volume_mounts + for enabled in [false, true] { + let names = SandboxRuntimeNames::new("pair"); + let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let pod = supervisor_pod( + "sandbox", + &names, + "pair", + "demo", + "gateway", + "supervisor:latest", + None, + "sandbox-sa", + 1000, + 1000, + &[], + "https://gateway:8080", + SupervisorClientTls::Disabled, + "{}", + "info", + 600, + None, + None, + None, + false, + false, + false, + None, + Some(("openshell-additional-ca-gateway-generation", digest)), + owner(), + &[], + ) + .expect("render supervisor Pod with destination trust"); + let pod = crate::app_listener::configure_supervisor_pod( + pod, + &crate::config::KubernetesAppListenerConfig { + tls_secret_name: "app-tls-secret".into(), + common: openshell_core::app_listener::AppListenerConfig { + enabled, + ..Default::default() + }, + ..Default::default() + }, + ); + let spec = pod.spec.expect("Pod spec"); + assert_eq!( + spec.volumes + .as_ref() + .unwrap() + .iter() + .any(|v| v.name == "app-tls"), + enabled + ); + let container = &spec.containers[0]; + assert!(container.command.as_deref().is_some_and(|command| { + command + .windows(2) + .any(|args| args == ["--additional-ca-bundle", ADDITIONAL_CA_MOUNT_PATH]) + })); + assert_eq!( + container + .volume_mounts + .as_deref() + .unwrap_or_default() + .iter() + .filter(|mount| mount.name == ADDITIONAL_CA_VOLUME_NAME) + .collect::>() + .as_slice(), + &[&VolumeMount { + name: ADDITIONAL_CA_VOLUME_NAME.to_string(), + mount_path: ADDITIONAL_CA_MOUNT_PATH.to_string(), + read_only: Some(true), + sub_path: Some(ADDITIONAL_CA_KEY.to_string()), + ..Default::default() + }] + ); + let volume = spec + .volumes .as_deref() .unwrap_or_default() .iter() - .filter(|mount| mount.name == ADDITIONAL_CA_VOLUME_NAME) - .collect::>() - .as_slice(), - &[&VolumeMount { - name: ADDITIONAL_CA_VOLUME_NAME.to_string(), - mount_path: ADDITIONAL_CA_MOUNT_PATH.to_string(), - read_only: Some(true), - sub_path: Some(ADDITIONAL_CA_KEY.to_string()), - ..Default::default() - }] - ); - let volume = spec - .volumes - .as_deref() - .unwrap_or_default() - .iter() - .find(|volume| volume.name == ADDITIONAL_CA_VOLUME_NAME) - .expect("destination trust ConfigMap volume"); - assert_eq!( - volume - .config_map - .as_ref() - .map(|source| source.name.as_str()), - Some("openshell-additional-ca-gateway-generation") - ); - assert_eq!( - volume - .config_map - .as_ref() - .and_then(|source| source.items.as_deref()) - .and_then(|items| items.first()) - .map(|item| (item.key.as_str(), item.path.as_str())), - Some((ADDITIONAL_CA_KEY, ADDITIONAL_CA_KEY)) - ); + .find(|volume| volume.name == ADDITIONAL_CA_VOLUME_NAME) + .expect("destination trust ConfigMap volume"); + assert_eq!( + volume + .config_map + .as_ref() + .map(|source| source.name.as_str()), + Some("openshell-additional-ca-gateway-generation") + ); + assert_eq!( + volume + .config_map + .as_ref() + .and_then(|source| source.items.as_deref()) + .and_then(|items| items.first()) + .map(|item| (item.key.as_str(), item.path.as_str())), + Some((ADDITIONAL_CA_KEY, ADDITIONAL_CA_KEY)) + ); + } } #[test] diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index 33ec8924a0..f5f2b62ab7 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -355,6 +355,7 @@ fn kubernetes_config( ) -> openshell_core::Result { let mut config: openshell_driver_kubernetes::KubernetesComputeConfig = context.driver_config()?; + config.sandbox_runtime.app_listener.common = context.app_listener_config(); config.sandbox_runtime_image = prefer_environment( config.sandbox_runtime_image, std::env::var(openshell_core::config::SANDBOX_RUNTIME_IMAGE_ENV).ok(), diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index c750794c1f..3fc93ef114 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -601,6 +601,15 @@ fn prepare_server_config_with_drivers( .map_err(|error| miette::miette!("invalid service routing configuration: {error}"))?; config.enable_sandbox_updates = args.enable_sandbox_updates; + + config.supervisor_app_listener = file + .as_ref() + .map(|f| f.openshell.supervisor.app_listener.clone()) + .unwrap_or_default(); + config + .supervisor_app_listener + .validate() + .map_err(|e| miette::miette!(e))?; if let Some(sources) = file .as_ref() .and_then(|file| file.openshell.gateway.provider_profile_sources.clone()) diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index bbe45b267e..bc09dae282 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -511,6 +511,8 @@ impl TryFrom for OtlpConfig { #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SupervisorFileSection { + #[serde(default)] + pub app_listener: openshell_core::app_listener::AppListenerConfig, /// Operator-owned external isolation routes, reachable by supervisors. #[serde(default)] pub isolation_backends: Vec, diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 279329c666..504c339690 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -3053,7 +3053,7 @@ pub async fn build_sandbox_config_snapshot_from_inputs( let settings = merge_effective_settings(global_settings, &sandbox_settings)?; let supervisor_middleware_services = state.middleware_registry.required_services(policy.as_ref()); - let config_revision = compute_config_revision_with_validation_mode( + let mut config_revision = compute_config_revision_with_validation_mode( policy.as_ref(), &settings, policy_source, @@ -3078,7 +3078,22 @@ pub async fn build_sandbox_config_snapshot_from_inputs( .filter(|_| matches!(policy_source, PolicySource::Global)), )?; + let service_routes = super::service::configured_app_routes(state, sandbox).await?; + // Include the allowlist in delivery identity so a service edit is not + // coalesced away as a duplicate policy/settings snapshot. + if !service_routes.is_empty() { + let mut bytes = config_revision.to_le_bytes().to_vec(); + for route in &service_routes { + route + .encode_length_delimited(&mut bytes) + .expect("encode service route"); + } + let hash = openshell_crypto::sha256(&bytes) + .map_err(|_| Status::internal("service configuration fingerprint failed"))?; + config_revision = u64::from_le_bytes(hash[..8].try_into().expect("hash prefix")); + } Ok(SandboxConfigSnapshot { + service_routes, configuration_instance_id: sandbox .status .as_ref() @@ -3113,6 +3128,7 @@ pub async fn build_sandbox_config_snapshot_from_inputs( fn sandbox_config_response(snapshot: SandboxConfigSnapshot) -> GetSandboxConfigResponse { GetSandboxConfigResponse { + service_routes: snapshot.service_routes, policy: snapshot.policy, version: snapshot.version, policy_hash: snapshot.policy_hash, diff --git a/crates/openshell-server/src/grpc/service.rs b/crates/openshell-server/src/grpc/service.rs index bf514da414..a47791e8d8 100644 --- a/crates/openshell-server/src/grpc/service.rs +++ b/crates/openshell-server/src/grpc/service.rs @@ -186,6 +186,9 @@ pub(super) async fn expose_service_endpoint( } }; + if state.config.supervisor_app_listener.enabled { + crate::config_delivery::publish_sandbox(state, sandbox.object_id()); + } let mut endpoint = endpoint; if let Some(ref mut meta) = endpoint.metadata { meta.resource_version = result.resource_version; @@ -360,6 +363,9 @@ pub(super) async fn handle_delete_service( state .service_route_index .remove(workspace, sandbox_name, &req.name); + if state.config.supervisor_app_listener.enabled { + crate::config_delivery::publish_sandbox(state, &endpoint.sandbox_id); + } service_routing::emit_service_endpoint_delete_event(&endpoint); } @@ -368,6 +374,56 @@ pub(super) async fn handle_delete_service( })) } +/// Complete bounded allowlist for the supervisor configuration. Disabled +/// ingress leaves existing gateway-only configurations and fingerprints alone. +pub(super) async fn configured_app_routes( + state: &ServerState, + sandbox: &Sandbox, +) -> Result, Status> { + if !state.config.supervisor_app_listener.enabled { + return Ok(Vec::new()); + } + let selector = format!("sandbox={}", sandbox.object_name()); + let page = state + .store + .list_message_page::( + ObjectListQuery::WorkspaceSelector { + workspace: sandbox.object_workspace(), + label_selector: &selector, + }, + None, + 1024, + ) + .await + .map_err(|_| Status::unavailable("service declarations unavailable"))?; + if page.next_cursor.is_some() { + return Err(Status::resource_exhausted( + "too many application service declarations", + )); + } + let mut service_routes: Vec<_> = page + .messages + .into_iter() + .filter(|ep| { + ep.sandbox_id == sandbox.object_id() + && ep.domain + && ep + .metadata + .as_ref() + .is_none_or(|m| m.deletion_time.is_none()) + }) + .map(|ep| openshell_core::proto::SandboxServiceRoute { + endpoint_id: ep.object_id().to_string(), + name: ep.name.clone(), + target_port: ep.target_port, + authorization_mode: ep.authorization_mode, + resource_version: ep.metadata.as_ref().map_or(0, |m| m.resource_version), + }) + .collect(); + service_routes.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(service_routes) +} + async fn get_service_endpoint( state: &Arc, workspace: &str, @@ -485,6 +541,121 @@ mod tests { validate_endpoint_name("service", "web-api", 28).unwrap(); } + #[tokio::test] + async fn service_declarations_update_configuration_identity_and_stay_sandbox_scoped() { + let mut state = test_server_state().await; + Arc::get_mut(&mut state) + .expect("unshared test state") + .config + .supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: 9443, + domain: "services.example.com".into(), + }; + seed_sandbox(&state, "first").await; + seed_sandbox(&state, "second").await; + let sandbox = state + .store + .get_message::("sandbox-first") + .await + .unwrap() + .unwrap(); + let before = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + for name in ["first", "second"] { + handle_expose_service( + &state, + authed_request(ExposeServiceRequest { + sandbox: name.into(), + name: "web".into(), + target_port: 8080, + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await + .unwrap(); + } + let installed = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + assert_eq!(installed.service_routes.len(), 1); + assert_eq!(installed.service_routes[0].target_port, 8080); + assert_ne!(installed.config_revision, before.config_revision); + + handle_expose_service( + &state, + authed_request(ExposeServiceRequest { + sandbox: "first".into(), + name: "web".into(), + target_port: 9090, + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await + .unwrap(); + let edited = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + assert_eq!(edited.service_routes[0].target_port, 9090); + assert_eq!( + edited.service_routes[0].endpoint_id, + installed.service_routes[0].endpoint_id + ); + assert_ne!(edited.config_revision, installed.config_revision); + + handle_delete_service( + &state, + authed_request(DeleteServiceRequest { + sandbox: "first".into(), + name: "web".into(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await + .unwrap(); + let removed = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + assert!(removed.service_routes.is_empty()); + assert_eq!(removed.config_revision, before.config_revision); + } + + #[tokio::test] + async fn gateway_only_exposure_does_not_change_supervisor_configuration() { + let state = test_server_state().await; + seed_sandbox(&state, "gateway-only").await; + let sandbox = state + .store + .get_message::("sandbox-gateway-only") + .await + .unwrap() + .unwrap(); + let before = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + handle_expose_service( + &state, + authed_request(ExposeServiceRequest { + sandbox: "gateway-only".into(), + name: "web".into(), + target_port: 8080, + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await + .unwrap(); + let after = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + assert!(after.service_routes.is_empty()); + assert_eq!(after.config_revision, before.config_revision); + } + #[test] fn validates_empty_optional_service_name() { validate_optional_endpoint_name("service", "", 28).unwrap(); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 12ca9223ea..655ed9bd12 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1716,6 +1716,14 @@ pub struct ComputeDriverConfigContext<'a> { } impl ComputeDriverConfigContext<'_> { + #[must_use] + pub fn app_listener_config(&self) -> openshell_core::app_listener::AppListenerConfig { + self.driver_startup + .file + .map(|file| file.openshell.supervisor.app_listener.clone()) + .unwrap_or_default() + } + #[must_use] pub fn driver_name(&self) -> &str { self.driver_name diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 08a2141884..5f7d87555b 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -178,7 +178,7 @@ mod tests { // image wire identity, now with explicit presence. Compute configuration // updates use a separate capability/service and add one classified RPC. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "be99bb3fcc7894eb85d774f54b642173e70dd37bc8e5fcaf3fdff2454ae689e1"; + "33e32bb6cb2251b3e75c6425555502e6c2fd675b640094437ef06856a46866e3"; const DURABLE_SCHEMA_SHA256: &str = "cb15f8f79a855064a43fe8c6cd908bf07eca6604e2bf8ab7d2de0ef33e97da4e"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = @@ -723,7 +723,7 @@ mod tests { overlap_hash.as_str(), ), ( - (334, 31), + (335, 31), (100, 22), (86, 22), PUBLIC_RPC_SCHEMA_SHA256, diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index 645b813604..efbffc5d35 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -29,6 +29,7 @@ openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-mid openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } openshell-supervisor-process = { path = "../openshell-supervisor-process" } +axum = { workspace = true } bytes = { workspace = true } clap = { workspace = true } futures = { workspace = true } @@ -70,10 +71,7 @@ bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] openshell-sandbox-backend = { path = "../openshell-sandbox-backend", features = ["test-support"] } openshell-otel-test-support = { path = "../openshell-otel-test-support" } bytes = { workspace = true } -http = { workspace = true } http-body-util = { workspace = true } -hyper = { workspace = true } -hyper-util = { workspace = true } tonic-prost = { workspace = true } tokio = { workspace = true, features = ["test-util"] } temp-env = "0.3" diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index 5956ef43f7..bbd59fbb56 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -257,3 +257,36 @@ cannot export each instance's records with that instance's collector headers and identity. Its console logs and each instance's gateway log push remain, and the Kubernetes driver warns at startup when the gateway configures supervisor export. + +## Application listener + +The opt-in application TLS listener serves declared HTTP/1.1 and WebSocket +services through each sandbox instance's `BoundaryLoopbackConnector`. The host +binds one listener and dispatches accepted TLS streams by SNI to a bounded queue +on the owning instance's runtime. That runtime owns request handling, credentials, +policy state, limits, and OCSF context. Stopping an instance unregisters only its +routes and closes its streams; it does not stop another instance's ingress. + +Gateway configuration snapshots carry the declared-service allowlist. Successful +poll and push application installs that allowlist; rejected configuration +suspends it. Polling renews a 30-second freshness lease. In push mode, an accepted +live supervisor session renews already installed, unexpired routes even when no +configuration changes. Rejection or a lease lapse cannot revive an old stream. +Ending the configuration loop suspends routes. + +`OPENSHELL_APP_LISTENER_ENABLED=true` explicitly enables the process listener. +`OPENSHELL_APP_LISTEN_ADDRESS`, `OPENSHELL_APP_SERVICE_DOMAIN`, +`OPENSHELL_APP_TLS_CERT`, and `OPENSHELL_APP_TLS_KEY` are then required. Hosted +instances receive the process router explicitly and never read another +instance's configuration from process environment. TLS configuration uses +`openshell-crypto`; certificates are read at process startup. Restart the +supervisor to reload application certificates. + +This path uses application or trusted-edge authentication independently from +gateway authentication. It validates Host against SNI, preserves public +Host/Origin and application cookies, strips gateway/edge identity material, and +retains service-specific bearer passthrough. It has no discovery API or public +URL selection. Dedicated and shared supervisors expose the app listener on their +Pod IPs without app Services. Clients must select the member owning their +sandbox. HA takeover may preserve the workload generation; existing ingress streams +are not migrated. Runtime ownership fencing must reject stale connectors. diff --git a/crates/openshell-supervisor/src/app_listener.rs b/crates/openshell-supervisor/src/app_listener.rs new file mode 100644 index 0000000000..a6ca598eb0 --- /dev/null +++ b/crates/openshell-supervisor/src/app_listener.rs @@ -0,0 +1,1446 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Opt-in application ingress, separate from the gateway control transport. + +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use axum::body::Body; +use http::{Request, Response, StatusCode, header}; +use hyper::body::Incoming; +use hyper_util::rt::{TokioIo, TokioTimer}; +use miette::{IntoDiagnostic, Result}; +use openshell_core::grpc_client::SettingsPollResult; +use openshell_core::proto::SandboxServiceRoute; +use openshell_isolation_interface::contract::LoopbackTarget; +use openshell_ocsf::{ + ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DispositionId, + HttpActivityBuilder, HttpResponse, NetworkActivityBuilder, SeverityId, StatusId, ocsf_emit, +}; +use openshell_sandbox_backend::BoundaryLoopbackConnector; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject}; +use std::sync::Mutex; +use tokio::sync::{Semaphore, mpsc, watch}; +use tokio::task::JoinSet; +use tokio::time::Instant; +use tokio_rustls::TlsAcceptor; + +const FRESHNESS: Duration = Duration::from_secs(30); +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + +pub struct Config { + address: SocketAddr, + domain: String, + certificate: PathBuf, + key: PathBuf, +} + +impl Config { + pub fn from_env() -> Result> { + Self::from_values(|name| std::env::var(name).ok()) + } + + fn from_values(read: impl Fn(&str) -> Option) -> Result> { + match read("OPENSHELL_APP_LISTENER_ENABLED").as_deref() { + None | Some("false") => return Ok(None), + Some("true") => {} + _ => { + return Err(miette::miette!( + "OPENSHELL_APP_LISTENER_ENABLED must be true or false" + )); + } + } + let address = read("OPENSHELL_APP_LISTEN_ADDRESS"); + let domain = read("OPENSHELL_APP_SERVICE_DOMAIN"); + let certificate = read("OPENSHELL_APP_TLS_CERT"); + let key = read("OPENSHELL_APP_TLS_KEY"); + let (Some(address), Some(domain), Some(certificate), Some(key)) = + (address, domain, certificate, key) + else { + return Err(miette::miette!( + "app listener requires address, domain, certificate, and key" + )); + }; + let address: SocketAddr = address.parse().into_diagnostic()?; + openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: address.port(), + domain: domain.clone(), + } + .validate() + .map_err(|error| miette::miette!(error))?; + Ok(Some(Self { + address, + domain, + certificate: certificate.into(), + key: key.into(), + })) + } + + pub fn bind(self) -> Result { + let certificates = CertificateDer::pem_file_iter(self.certificate) + .into_diagnostic()? + .collect::, _>>() + .into_diagnostic()?; + let key = PrivateKeyDer::from_pem_file(self.key).into_diagnostic()?; + let mut tls = openshell_crypto::tls::try_server_builder() + .into_diagnostic()? + .with_no_client_auth() + .with_single_cert(certificates, key) + .into_diagnostic()?; + tls.alpn_protocols = vec![b"http/1.1".to_vec()]; + let socket = std::net::TcpListener::bind(self.address).into_diagnostic()?; + socket.set_nonblocking(true).into_diagnostic()?; + let listener = tokio::net::TcpListener::from_std(socket).into_diagnostic()?; + Ok(BoundListener { + listener, + acceptor: TlsAcceptor::from(Arc::new(tls)), + domain: self.domain, + }) + } +} + +fn valid_label(label: &str) -> bool { + !label.is_empty() + && label.len() <= 63 + && !label.starts_with('-') + && !label.ends_with('-') + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') +} + +#[derive(Clone, Default)] +struct Snapshot { + routes: BTreeMap, + generations: BTreeMap, + next_generation: u64, + expires: Option, + workspace: String, + // Whole-sandbox suspension fence, independent of individual service edits. + epoch: u64, +} + +#[derive(Clone, Copy)] +struct RouteToken { + epoch: u64, + generation: u64, +} + +/// Shared by policy reconciliation and ingress. Only accepted configuration +/// refreshes this lease; route changes cannot restart or replace the workload. +#[derive(Clone)] +pub struct Routes { + registration: Option>, + sender: watch::Sender, + stopped: Arc, +} + +impl Default for Routes { + fn default() -> Self { + Self { + sender: watch::channel(Snapshot::default()).0, + stopped: Arc::new(AtomicBool::new(false)), + registration: None, + } + } +} + +pub(crate) struct ConfigurationGuard(Routes); +impl Drop for ConfigurationGuard { + fn drop(&mut self) { + self.0.suspend(); + } +} + +impl Routes { + pub(crate) fn configuration_guard(self) -> ConfigurationGuard { + ConfigurationGuard(self) + } + + fn valid(&self, route: &SandboxServiceRoute, token: RouteToken) -> bool { + let snapshot = self.sender.borrow(); + snapshot + .expires + .is_some_and(|expiry| expiry > Instant::now()) + && snapshot.epoch == token.epoch + && snapshot.generations.get(&route.name) == Some(&token.generation) + && snapshot.routes.get(&route.name) == Some(route) + } + pub(crate) fn install(&self, result: &SettingsPollResult) { + if self.stopped.load(Ordering::Acquire) { + return; + } + let mut routes = BTreeMap::new(); + if !result.configuration_admitted { + self.suspend(); + return; + } + if result.configuration_admitted + && valid_label(&result.workspace) + && !result.workspace.contains("--") + { + for route in &result.service_routes { + if route.endpoint_id.is_empty() + || route.target_port == 0 + || route.target_port > 65535 + || !matches!(route.authorization_mode, 0..=2) + || (!route.name.is_empty() + && (!valid_label(&route.name) || route.name.contains("--"))) + { + self.suspend(); + return; + } + if routes.insert(route.name.clone(), route.clone()).is_some() { + self.suspend(); + return; + } + } + } + if routes.len() > 1024 || !valid_label(&result.workspace) || result.workspace.contains("--") + { + self.suspend(); + return; + } + if let Some(registration) = &self.registration + && !registration.bind(&result.workspace) + { + self.suspend(); + return; + } + let changed = self.sender.borrow().routes != routes; + self.sender.send_modify(|snapshot| { + // Renewal after a lapse cannot revive a connection whose lease expired. + if snapshot + .expires + .is_some_and(|expiry| expiry <= Instant::now()) + { + snapshot.epoch = snapshot.epoch.wrapping_add(1); + } + let generations = routes + .iter() + .map(|(name, route)| { + let previous = (snapshot.routes.get(name) == Some(route)) + .then(|| snapshot.generations.get(name).copied()) + .flatten(); + let generation = previous.unwrap_or_else(|| { + snapshot.next_generation = snapshot + .next_generation + .checked_add(1) + .expect("app route generation exhausted"); + snapshot.next_generation + }); + (name.clone(), generation) + }) + .collect(); + if snapshot.workspace != result.workspace { + snapshot.epoch = snapshot.epoch.wrapping_add(1); + } + snapshot.workspace.clone_from(&result.workspace); + snapshot.routes = routes; + snapshot.generations = generations; + snapshot.expires = Some(Instant::now() + FRESHNESS); + }); + if changed { + ocsf_emit!( + ConfigStateChangeBuilder::new(crate::ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .message("App listener service routes updated") + .build() + ); + } + } + + /// Renew only already installed routes while a live push session supplies + /// freshness. A rejected configuration is suspended and cannot be revived. + pub(crate) fn renew(&self) { + if self.stopped.load(Ordering::Acquire) { + return; + } + self.sender.send_modify(|snapshot| { + if snapshot + .expires + .is_some_and(|expiry| expiry > Instant::now()) + { + snapshot.expires = Some(Instant::now() + FRESHNESS); + } + }); + } + + pub(crate) fn suspend(&self) { + self.sender.send_modify(|snapshot| { + snapshot.routes.clear(); + snapshot.generations.clear(); + snapshot.expires = None; + snapshot.epoch = snapshot.epoch.wrapping_add(1); + }); + } + + fn resolve( + &self, + sni: &str, + prefix: &str, + domain: &str, + ) -> Option<(SandboxServiceRoute, RouteToken)> { + let encoded = sni.strip_suffix(&format!(".{domain}"))?; + let service = if encoded == prefix { + "" + } else { + encoded.strip_prefix(&format!("{prefix}--"))? + }; + let snapshot = self.sender.borrow(); + if snapshot.expires? <= Instant::now() { + return None; + } + Some(( + snapshot.routes.get(service)?.clone(), + RouteToken { + epoch: snapshot.epoch, + generation: *snapshot.generations.get(service)?, + }, + )) + } + + async fn revoked(&self, route: &SandboxServiceRoute, token: RouteToken) { + let mut receiver = self.sender.subscribe(); + loop { + let expiry = { + let snapshot = receiver.borrow_and_update(); + if snapshot.epoch != token.epoch + || snapshot.generations.get(&route.name) != Some(&token.generation) + || snapshot.routes.get(&route.name) != Some(route) + { + return; + } + let Some(expiry) = snapshot.expires else { + return; + }; + expiry + }; + tokio::select! { + () = tokio::time::sleep_until(expiry) => return, + result = receiver.changed() => if result.is_err() { return; }, + } + } + } +} + +pub struct BoundListener { + listener: tokio::net::TcpListener, + acceptor: TlsAcceptor, + domain: String, +} + +/// Host-level dispatcher. It never shares a sandbox connector or policy +/// between instances: TLS streams are handed to the owning instance runtime. +#[derive(Clone, Default)] +pub struct Router { + domain: String, + registrations: Arc>>>, +} + +struct Accepted { + tls: tokio_rustls::server::TlsStream, + permit: tokio::sync::OwnedSemaphorePermit, +} + +struct Registration { + router: Router, + sandbox: String, + prefix: Mutex, + sender: mpsc::Sender, +} + +impl Registration { + fn bind(&self, workspace: &str) -> bool { + let next = format!("{workspace}--{}", self.sandbox); + let mut prefix = self + .prefix + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut entries = self + .router + .registrations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if entries + .get(&next) + .is_some_and(|sender| !sender.same_channel(&self.sender)) + { + return false; + } + if *prefix != next { + if entries + .get(&*prefix) + .is_some_and(|s| s.same_channel(&self.sender)) + { + entries.remove(&*prefix); + } + prefix.clone_from(&next); + } + entries.insert(next, self.sender.clone()); + true + } + + fn remove(&self) { + let prefix = self + .prefix + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut entries = self + .router + .registrations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if entries + .get(&*prefix) + .is_some_and(|s| s.same_channel(&self.sender)) + { + entries.remove(&*prefix); + } + } +} + +/// Process-owned listener. Dropping it stops acceptance without affecting +/// another host process. Individual instance lifetimes are independent. +pub struct HostHandle { + task: tokio::task::JoinHandle<()>, + router: Router, +} + +impl HostHandle { + #[must_use] + pub fn router(&self) -> Router { + self.router.clone() + } +} + +impl Drop for HostHandle { + fn drop(&mut self) { + self.task.abort(); + } +} + +pub struct InstanceListener { + pub(crate) routes: Routes, + receiver: mpsc::Receiver, + domain: String, +} + +pub struct Handle { + task: tokio::task::JoinHandle<()>, + routes: Routes, +} + +impl Drop for Handle { + fn drop(&mut self) { + self.routes.stopped.store(true, Ordering::Release); + self.routes.suspend(); + if let Some(registration) = &self.routes.registration { + registration.remove(); + } + self.task.abort(); + } +} + +impl Router { + pub(crate) fn instance(&self, sandbox: &str) -> Result { + if !valid_label(sandbox) || sandbox.contains("--") { + return Err(miette::miette!( + "app listener requires a canonical sandbox name" + )); + } + let (sender, receiver) = mpsc::channel(16); + let registration = Arc::new(Registration { + router: self.clone(), + sandbox: sandbox.to_string(), + prefix: Mutex::new(String::new()), + sender, + }); + Ok(InstanceListener { + routes: Routes { + registration: Some(registration), + ..Default::default() + }, + receiver, + domain: self.domain.clone(), + }) + } +} + +impl BoundListener { + #[must_use] + pub fn start(self) -> HostHandle { + let router = Router { + domain: self.domain.clone(), + ..Default::default() + }; + let dispatch = router.clone(); + let task = tokio::spawn(async move { + let permits = Arc::new(Semaphore::new(1024)); + let mut connections = JoinSet::new(); + loop { + tokio::select! { + Some(_) = connections.join_next(), if !connections.is_empty() => {}, + accepted = self.listener.accept() => { + let Ok((stream, _)) = accepted else { break; }; + let Ok(permit) = permits.clone().try_acquire_owned() else { continue; }; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + let acceptor = self.acceptor.clone(); + let domain = self.domain.clone(); + let dispatch = dispatch.clone(); + connections.spawn(async move { + let Ok(Ok(tls)) = tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await else { return; }; + let Some(sni) = tls.get_ref().1.server_name() else { return; }; + let Some(label) = sni.strip_suffix(&format!(".{domain}")) else { return; }; + let mut segments = label.split("--"); + let (Some(workspace), Some(sandbox)) = (segments.next(), segments.next()) else { return; }; + let prefix = format!("{workspace}--{sandbox}"); + let sender = dispatch.registrations.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&prefix).cloned(); + if let Some(sender) = sender { let _ = sender.try_send(Accepted { tls, permit }); } + }); + } + } + } + }); + HostHandle { task, router } + } +} + +struct Permits { + _host: tokio::sync::OwnedSemaphorePermit, + _instance: tokio::sync::OwnedSemaphorePermit, +} + +impl InstanceListener { + /// Called on the sandbox instance runtime, preserving its OCSF/logging + /// identity even when multiple sandboxes share one listener process. + pub(crate) fn start(mut self, connector: Arc) -> Handle { + let retained_routes = self.routes.clone(); + let task = tokio::spawn(async move { + let routes = self.routes.clone(); + let permits = Arc::new(Semaphore::new(64)); + let mut connections = JoinSet::new(); + ocsf_emit!( + AppLifecycleBuilder::new(crate::ocsf_ctx()) + .activity(ActivityId::Open) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .message("Supervisor app ingress ready") + .build() + ); + loop { + tokio::select! { + Some(_) = connections.join_next(), if !connections.is_empty() => {}, + accepted = self.receiver.recv() => { + let Some(Accepted { tls, permit: host_permit }) = accepted else { break; }; + let Ok(instance_permit) = permits.clone().try_acquire_owned() else { continue; }; + let Some(sni) = tls.get_ref().1.server_name().map(str::to_ascii_lowercase) else { continue; }; + let prefix = routes.registration.as_ref().expect("instance registration").prefix.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone(); + let Some((route, token)) = routes.resolve(&sni, &prefix, &self.domain) else { + emit_denial("App listener unknown or expired SNI route"); continue; + }; + let routes = routes.clone(); + let connector = connector.clone(); + connections.spawn(async move { + let connection_routes = routes.clone(); + let connection_route = route.clone(); + let permit = Arc::new(Permits { _host: host_permit, _instance: instance_permit }); + let service = hyper::service::service_fn(move |req| { + proxy(req, sni.clone(), route.clone(), routes.clone(), connector.clone(), permit.clone(), token) + }); + let connection = hyper::server::conn::http1::Builder::new() + .timer(TokioTimer::new()).header_read_timeout(HANDSHAKE_TIMEOUT) + .max_buf_size(65536).max_headers(64) + .serve_connection(TokioIo::new(tls), service).with_upgrades(); + tokio::select! { + _ = connection => {}, + () = connection_routes.revoked(&connection_route, token) => {}, + } + }); + } + } + } + }); + Handle { + task, + routes: retained_routes, + } + } +} + +fn error(status: StatusCode) -> Response { + ocsf_emit!( + HttpActivityBuilder::new(crate::ocsf_ctx()) + .http_response(HttpResponse { + code: status.as_u16() + }) + .activity(ActivityId::Other) + .action(ActionId::Denied) + .disposition(if status.is_server_error() { + DispositionId::Error + } else { + DispositionId::Blocked + }) + .severity(if status.is_server_error() { + SeverityId::Low + } else { + SeverityId::Medium + }) + .status(StatusId::Failure) + .message(format!( + "App listener request rejected [status:{}]", + status.as_u16() + )) + .build() + ); + Response::builder() + .status(status) + .body(Body::empty()) + .expect("valid error response") +} + +fn emit_denial(message: &str) { + ocsf_emit!( + NetworkActivityBuilder::new(crate::ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(openshell_ocsf::Endpoint { + domain: None, + ip: None, + port: None, + name: None, + svc_name: None, + }) + .message(message) + .build() + ); +} + +fn authority_matches(req: &Request, sni: &str) -> bool { + let mut values = req.headers().get_all(header::HOST).iter(); + let Some(value) = values.next() else { + return false; + }; + if values.next().is_some() { + return false; + } + let Ok(value) = value.to_str() else { + return false; + }; + if value.contains('@') { + return false; + } + let Ok(authority) = value.parse::() else { + return false; + }; + if authority.port().is_some() && authority.port_u16().is_none() { + return false; + } + authority.host().eq_ignore_ascii_case(sni) + && req + .uri() + .authority() + .is_none_or(|uri| uri.as_str().eq_ignore_ascii_case(value)) +} + +fn gateway_header(name: &str) -> bool { + matches!( + name, + "proxy-authorization" + | "cf-access-jwt-assertion" + | "cf-access-authenticated-user-email" + | "x-openshell-principal" + | "x-openshell-user" + | "x-openshell-roles" + | "x-forwarded-client-cert" + | "x-ssl-client-cert" + | "x-client-cert" + ) +} + +fn hop_header(name: &str) -> bool { + matches!( + name, + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + ) +} + +fn sanitize_headers(headers: &mut http::HeaderMap, passthrough: bool, websocket: bool) -> bool { + if passthrough { + let values: Vec<_> = headers.get_all(header::AUTHORIZATION).iter().collect(); + if values.len() > 1 { + return false; + } + if let Some(value) = values.first() { + let Ok(value) = value.to_str() else { + return false; + }; + let Some(token) = openshell_core::auth::strip_bearer_scheme(value) else { + return false; + }; + if token.is_empty() + || !token + .trim_end_matches('=') + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-._~+/".contains(&byte)) + || token.trim_end_matches('=').is_empty() + { + return false; + } + } + } else { + headers.remove(header::AUTHORIZATION); + } + let connection_headers: Vec = headers + .get_all(header::CONNECTION) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .map(|value| value.trim().to_ascii_lowercase()) + .collect(); + let remove: Vec<_> = headers + .keys() + .filter(|name| { + gateway_header(name.as_str()) + || ((hop_header(name.as_str()) + || connection_headers + .iter() + .any(|value| value == name.as_str())) + && !(websocket && matches!(name.as_str(), "connection" | "upgrade"))) + }) + .cloned() + .collect(); + for name in remove { + headers.remove(name); + } + let cookies: Vec<_> = headers + .get_all(header::COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(';')) + .filter(|cookie| { + cookie.split_once('=').is_some_and(|(name, _)| { + !name.trim().eq_ignore_ascii_case("CF_Authorization") + && !name.trim().eq_ignore_ascii_case("cf-authorization") + }) + }) + .map(str::trim) + .collect(); + let cookie = cookies.join("; "); + headers.remove(header::COOKIE); + if !cookie.is_empty() + && let Ok(cookie) = cookie.parse() + { + headers.insert(header::COOKIE, cookie); + } + true +} + +async fn proxy( + mut req: Request, + sni: String, + route: SandboxServiceRoute, + routes: Routes, + connector: Arc, + permit: Arc, + token: RouteToken, +) -> std::result::Result, std::convert::Infallible> { + if !routes.valid(&route, token) { + return Ok(error(StatusCode::SERVICE_UNAVAILABLE)); + } + if !authority_matches(&req, &sni) { + return Ok(error(StatusCode::MISDIRECTED_REQUEST)); + } + // Keep the public authority after validating it against SNI. The dial target + // is loopback, but rewriting Host would break application same-origin checks. + let authority = req.headers()[header::HOST].clone(); + let websocket = req + .headers() + .get(header::UPGRADE) + .is_some_and(|value| value.as_bytes().eq_ignore_ascii_case(b"websocket")); + if !sanitize_headers(req.headers_mut(), route.authorization_mode == 2, websocket) { + return Ok(error(StatusCode::BAD_REQUEST)); + } + let downstream = websocket.then(|| hyper::upgrade::on(&mut req)); + let target = LoopbackTarget::new( + "127.0.0.1".parse().expect("loopback"), + u16::try_from(route.target_port).expect("validated route port"), + ) + .expect("validated route port"); + let Ok(Ok(stream)) = tokio::time::timeout(CONNECT_TIMEOUT, connector.connect(target)).await + else { + return Ok(error(StatusCode::BAD_GATEWAY)); + }; + let Ok((mut sender, connection)) = + hyper::client::conn::http1::handshake(TokioIo::new(stream)).await + else { + return Ok(error(StatusCode::BAD_GATEWAY)); + }; + let upstream_task = tokio::spawn(async move { + let _ = connection.with_upgrades().await; + }); + let (mut parts, body) = req.into_parts(); + parts.uri = parts + .uri + .path_and_query() + .map_or("/", |path| path.as_str()) + .parse() + .expect("valid path"); + parts.headers.insert(header::HOST, authority); + let response = sender.send_request(Request::from_parts(parts, body)).await; + let Ok(mut response) = response else { + upstream_task.abort(); + return Ok(error(StatusCode::BAD_GATEWAY)); + }; + if response.status() == StatusCode::SWITCHING_PROTOCOLS && websocket { + let upstream = hyper::upgrade::on(&mut response); + let downstream = downstream.expect("websocket upgrade"); + tokio::spawn(async move { + let _permit = permit; + tokio::select! { + () = async { + if let (Ok(upstream), Ok(downstream)) = tokio::join!(upstream, downstream) { + let _ = tokio::io::copy_bidirectional(&mut TokioIo::new(upstream), &mut TokioIo::new(downstream)).await; + } + } => {}, + () = routes.revoked(&route, token) => {}, + } + upstream_task.abort(); + }); + let (parts, _) = response.into_parts(); + return Ok(Response::from_parts(parts, Body::empty())); + } + let (parts, body) = response.into_parts(); + Ok(Response::from_parts(parts, Body::new(body))) +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::{SinkExt as _, StreamExt as _}; + use openshell_isolation_interface::contract::{BackendError, BoundaryDuplexStream}; + use openshell_sandbox_backend::test_support as rpc_test; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio_tungstenite::tungstenite::client::IntoClientRequest as _; + + #[test] + fn listener_requires_explicit_enablement_and_complete_configuration() { + let complete = BTreeMap::from([ + ("OPENSHELL_APP_LISTEN_ADDRESS", "0.0.0.0:9443"), + ("OPENSHELL_APP_SERVICE_DOMAIN", "services.example.com"), + ("OPENSHELL_APP_TLS_CERT", "/operator/cert.pem"), + ("OPENSHELL_APP_TLS_KEY", "/operator/key.pem"), + ]); + assert!( + Config::from_values(|name| complete.get(name).map(ToString::to_string)) + .unwrap() + .is_none() + ); + let mut values = complete.clone(); + values.insert("OPENSHELL_APP_LISTENER_ENABLED", "false"); + assert!( + Config::from_values(|name| values.get(name).map(ToString::to_string)) + .unwrap() + .is_none() + ); + values.insert("OPENSHELL_APP_LISTENER_ENABLED", "true"); + assert_eq!( + Config::from_values(|name| values.get(name).map(ToString::to_string)) + .unwrap() + .unwrap() + .address + .port(), + 9443 + ); + values.remove("OPENSHELL_APP_TLS_KEY"); + assert!(Config::from_values(|name| values.get(name).map(ToString::to_string)).is_err()); + assert!( + Config::from_values( + |name| (name == "OPENSHELL_APP_LISTENER_ENABLED").then(|| "true".into()) + ) + .is_err() + ); + } + + const HOST: &str = "default--sandbox-a--web.svc.openshell.internal"; + + fn installed(routes: Vec) -> SettingsPollResult { + SettingsPollResult { + service_routes: routes, + settings_revision: 1, + configuration_instance_id: "runtime-a".into(), + configuration_admitted: true, + configuration_error: String::new(), + policy: None, + version: 1, + policy_hash: String::new(), + config_revision: 1, + policy_source: openshell_core::proto::PolicySource::Unspecified, + settings: std::collections::HashMap::default(), + global_policy_version: 0, + provider_env_revision: 0, + provider_attachment_epoch: String::new(), + supervisor_middleware_services: Vec::new(), + workspace: "default".into(), + policy_validation_failure_mode: openshell_core::PolicyValidationFailureMode::default(), + extension_authentication_enabled: false, + } + } + + struct Connector { + calls: AtomicUsize, + } + + #[tonic::async_trait] + impl rpc_test::BoundaryLoopbackConnector for Connector { + async fn connect( + &self, + target: LoopbackTarget, + ) -> std::result::Result { + assert_eq!(target.port(), 8080); + let (client, server) = tokio::io::duplex(8192); + self.calls.fetch_add(1, Ordering::SeqCst); + tokio::spawn(async move { + let service = hyper::service::service_fn(|mut req: Request| async move { + if let Some(key) = req.headers().get("sec-websocket-key") { + let accept = tokio_tungstenite::tungstenite::handshake::derive_accept_key( + key.as_bytes(), + ); + let upgraded = hyper::upgrade::on(&mut req); + tokio::spawn(async move { + if let Ok(stream) = upgraded.await { + let mut ws = tokio_tungstenite::WebSocketStream::from_raw_socket( + TokioIo::new(stream), + tokio_tungstenite::tungstenite::protocol::Role::Server, + None, + ) + .await; + while let Some(Ok(message)) = ws.next().await { + if ws.send(message).await.is_err() { + break; + } + } + } + }); + return Ok::<_, std::convert::Infallible>( + Response::builder() + .status(101) + .header("connection", "upgrade") + .header("upgrade", "websocket") + .header("sec-websocket-accept", accept) + .header("x-test-upstream-host", req.headers()[header::HOST].clone()) + .header( + "x-test-upstream-origin", + req.headers() + .get(header::ORIGIN) + .cloned() + .unwrap_or_else(|| "absent".parse().unwrap()), + ) + .body(Body::empty()) + .unwrap(), + ); + } + let body = format!( + "{} {} {} {}", + req.uri(), + req.headers()[header::HOST].to_str().unwrap(), + req.headers() + .get(header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .unwrap_or("stripped"), + req.headers() + .get(header::ORIGIN) + .and_then(|v| v.to_str().ok()) + .unwrap_or("absent") + ); + Ok::<_, std::convert::Infallible>(Response::new(Body::from(body))) + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(TokioIo::new(server), service) + .with_upgrades() + .await; + }); + Ok(Box::new(client)) + } + } + + struct Fixture { + _directory: tempfile::TempDir, + handle: Handle, + address: SocketAddr, + tls: Arc, + routes: Routes, + connector: Arc, + route: SandboxServiceRoute, + host: HostHandle, + } + + fn fixture() -> Fixture { + openshell_crypto::tls::ensure_default_provider(); + let directory = tempfile::tempdir().unwrap(); + let cert = openshell_crypto::pki::generate_simple_self_signed(vec![ + "*.svc.openshell.internal".into(), + ]) + .unwrap(); + let cert_path = directory.path().join("cert.pem"); + let key_path = directory.path().join("key.pem"); + std::fs::write(&cert_path, cert.cert.pem()).unwrap(); + std::fs::write(&key_path, cert.key_pair.serialize_pem().unwrap()).unwrap(); + let config = Config { + address: "127.0.0.1:0".parse().unwrap(), + domain: "svc.openshell.internal".into(), + certificate: cert_path, + key: key_path, + }; + let bound = config.bind().unwrap(); + let address = bound.listener.local_addr().unwrap(); + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.cert.der().clone()).unwrap(); + let tls = Arc::new( + openshell_crypto::tls::client_builder() + .with_root_certificates(roots) + .with_no_client_auth(), + ); + let host = bound.start(); + let instance = host.router().instance("sandbox-a").unwrap(); + let routes = instance.routes.clone(); + let route = SandboxServiceRoute { + endpoint_id: "endpoint-a".into(), + name: "web".into(), + target_port: 8080, + authorization_mode: 1, + resource_version: 1, + }; + routes.install(&installed(vec![route.clone()])); + let connector = Arc::new(Connector { + calls: AtomicUsize::new(0), + }); + let handle = instance.start(rpc_test::loopback(connector.clone())); + Fixture { + _directory: directory, + handle, + address, + tls, + routes, + connector, + route, + host, + } + } + + async fn request(f: &Fixture, sni: &str, host: &str, extra: &str) -> String { + request_method(f, "GET", sni, host, extra).await + } + + async fn request_method( + f: &Fixture, + method: &str, + sni: &str, + host: &str, + extra: &str, + ) -> String { + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[f.address]) + .await + .unwrap(); + let server_name = rustls::pki_types::ServerName::try_from(sni.to_string()).unwrap(); + let mut tls = tokio_rustls::TlsConnector::from(f.tls.clone()) + .connect(server_name, stream) + .await + .unwrap(); + let _ = tls + .write_all( + format!( + "{method} /hello?x=1 HTTP/1.1\r\nHost: {host}\r\nConnection: close\r\n{extra}\r\n" + ) + .as_bytes(), + ) + .await; + let mut response = Vec::new(); + // Unknown SNI closes the TLS stream without an HTTP response. + let _ = tokio::time::timeout(Duration::from_secs(3), tls.read_to_end(&mut response)) + .await + .unwrap(); + String::from_utf8(response).unwrap() + } + + #[tokio::test] + async fn browser_post_preserves_public_authority_and_origin() { + let f = fixture(); + let authority = format!("{HOST}:18443"); + let origin = format!("https://{authority}"); + let response = request_method( + &f, + "POST", + HOST, + &authority, + &format!("Origin: {origin}\r\nContent-Length: 0\r\n"), + ) + .await; + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + assert!( + response.contains(&format!("/hello?x=1 {authority} stripped {origin}")), + "{response}" + ); + } + + #[tokio::test] + async fn shared_listener_keeps_instance_connectors_and_lifetimes_separate() { + let f = fixture(); + let second = f.host.router().instance("sandbox-b").unwrap(); + let second_routes = second.routes.clone(); + let second_connector = Arc::new(Connector { + calls: AtomicUsize::new(0), + }); + let second_handle = second.start(rpc_test::loopback(second_connector.clone())); + second_routes.install(&installed(vec![SandboxServiceRoute { + endpoint_id: "endpoint-b".into(), + ..f.route.clone() + }])); + let host_b = "default--sandbox-b--web.svc.openshell.internal"; + assert!( + request(&f, host_b, host_b, "") + .await + .starts_with("HTTP/1.1 200") + ); + assert_eq!(second_connector.calls.load(Ordering::SeqCst), 1); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 0); + + drop(second_handle); + assert!( + !request(&f, host_b, host_b, "") + .await + .starts_with("HTTP/1.1 200") + ); + assert!( + request(&f, HOST, HOST, "") + .await + .starts_with("HTTP/1.1 200") + ); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn stale_instance_cleanup_cannot_remove_its_replacement() { + let f = fixture(); + let replacement = f.host.router().instance("sandbox-a").unwrap(); + replacement + .routes + .install(&installed(vec![f.route.clone()])); + assert!( + replacement.routes.sender.borrow().routes.is_empty(), + "duplicate instance must not claim a live registration" + ); + let old_routes = f.routes.clone(); + drop(f.handle); + replacement + .routes + .install(&installed(vec![f.route.clone()])); + let prefix = "default--sandbox-a"; + assert!( + f.host + .router() + .registrations + .lock() + .unwrap() + .contains_key(prefix) + ); + // Simulate delayed cleanup from the old runtime after replacement. + old_routes.registration.as_ref().unwrap().remove(); + assert!( + f.host + .router() + .registrations + .lock() + .unwrap() + .contains_key(prefix) + ); + } + + #[tokio::test(start_paused = true)] + async fn push_freshness_renews_live_routes_but_cannot_revive_rejected_routes() { + let routes = Routes::default(); + let route = SandboxServiceRoute { + endpoint_id: "ep".into(), + name: "web".into(), + target_port: 8080, + authorization_mode: 1, + resource_version: 1, + }; + routes.install(&installed(vec![route.clone()])); + for _ in 0..10 { + tokio::time::advance(Duration::from_secs(5)).await; + routes.renew(); + } + assert!( + routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .is_some() + ); + let mut rejected = installed(vec![route]); + rejected.configuration_admitted = false; + routes.install(&rejected); + routes.renew(); + assert!( + routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .is_none() + ); + } + + #[tokio::test] + async fn tls_routes_only_matching_sni_and_authority() { + let f = fixture(); + let response = request( + &f, + HOST, + &format!("{HOST}:18443"), + "Authorization: Bearer app-token\r\n", + ) + .await; + assert!(response.starts_with("HTTP/1.1 200"), "{response}"); + assert!( + response.contains(&format!("/hello?x=1 {HOST}:18443 stripped")), + "{response}" + ); + let mismatch = request( + &f, + HOST, + "default--sandbox-b--web.svc.openshell.internal", + "", + ) + .await; + assert!(mismatch.starts_with("HTTP/1.1 421"), "{mismatch}"); + let unknown = request( + &f, + "default--sandbox-b--web.svc.openshell.internal", + "default--sandbox-b--web.svc.openshell.internal", + "", + ) + .await; + assert!(unknown.is_empty()); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn duplicate_host_and_malformed_bearer_never_dial_boundary() { + let f = fixture(); + let duplicate = request(&f, HOST, HOST, &format!("Host: {HOST}\r\n")).await; + assert!(!duplicate.starts_with("HTTP/1.1 200")); + f.routes + .sender + .send_modify(|snapshot| snapshot.routes.get_mut("web").unwrap().authorization_mode = 2); + let malformed = request(&f, HOST, HOST, "Authorization: Basic credential\r\n").await; + assert!(malformed.starts_with("HTTP/1.1 400")); + let valid = request(&f, HOST, HOST, "Authorization: bEaReR app-token\r\n").await; + assert!(valid.contains("bEaReR app-token"), "{valid}"); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn withdrawn_and_expired_routes_never_dial_boundary() { + let f = fixture(); + let (_, old_token) = f + .routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .unwrap(); + f.routes.suspend(); + f.routes.install(&installed(vec![f.route.clone()])); + // Even a quickly restored identical route cannot revive an old stream. + tokio::time::timeout( + Duration::from_secs(1), + f.routes.revoked(&f.route, old_token), + ) + .await + .unwrap(); + f.routes.sender.send_modify(|snapshot| { + snapshot.expires = Some(Instant::now() - Duration::from_secs(1)); + }); + assert!(request(&f, HOST, HOST, "").await.is_empty()); + f.routes.suspend(); + assert!(request(&f, HOST, HOST, "").await.is_empty()); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 0); + } + + #[test] + fn headers_strip_edge_identity_and_connection_named_fields() { + let mut headers = http::HeaderMap::new(); + headers.insert( + header::COOKIE, + "app=session; CF_Authorization=secret".parse().unwrap(), + ); + headers.insert("x-forwarded-client-cert", "secret".parse().unwrap()); + headers.insert(header::CONNECTION, "x-internal".parse().unwrap()); + headers.insert("x-internal", "secret".parse().unwrap()); + assert!(sanitize_headers(&mut headers, false, false)); + assert_eq!(headers[header::COOKIE], "app=session"); + assert!(!headers.contains_key("x-forwarded-client-cert")); + assert!(!headers.contains_key("x-internal")); + } + #[tokio::test] + async fn websocket_echo_closes_on_route_withdrawal() { + let f = fixture(); + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[f.address]) + .await + .unwrap(); + let name = rustls::pki_types::ServerName::try_from(HOST.to_string()).unwrap(); + let tls = tokio_rustls::TlsConnector::from(f.tls.clone()) + .connect(name, stream) + .await + .unwrap(); + let mut request = format!("wss://{HOST}/echo").into_client_request().unwrap(); + request + .headers_mut() + .insert(header::ORIGIN, format!("https://{HOST}").parse().unwrap()); + let (mut ws, response) = tokio_tungstenite::client_async(request, tls).await.unwrap(); + assert_eq!(response.headers()["x-test-upstream-host"], HOST); + assert_eq!( + response.headers()["x-test-upstream-origin"], + format!("https://{HOST}") + ); + ws.send(tokio_tungstenite::tungstenite::Message::Text( + "hello".into(), + )) + .await + .unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .unwrap() + .unwrap() + .unwrap() + .to_text() + .unwrap(), + "hello" + ); + f.routes.suspend(); + let closed = tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .unwrap(); + assert!(closed.is_none() || closed.unwrap().is_err()); + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 1); + } + + type TestWebSocket = + tokio_tungstenite::WebSocketStream>; + + async fn websocket(f: &Fixture, service: &str) -> TestWebSocket { + let host = format!("default--sandbox-a--{service}.svc.openshell.internal"); + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[f.address]) + .await + .unwrap(); + let name = rustls::pki_types::ServerName::try_from(host.clone()).unwrap(); + let tls = tokio_rustls::TlsConnector::from(f.tls.clone()) + .connect(name, stream) + .await + .unwrap(); + tokio_tungstenite::client_async(format!("wss://{host}/echo"), tls) + .await + .unwrap() + .0 + } + + async fn echo(ws: &mut TestWebSocket, value: &str) { + ws.send(tokio_tungstenite::tungstenite::Message::Text(value.into())) + .await + .unwrap(); + let echoed = tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(echoed.to_text().unwrap(), value); + } + + async fn closed(ws: &mut TestWebSocket) { + let next = tokio::time::timeout(Duration::from_secs(2), ws.next()) + .await + .expect("revoked WebSocket must close"); + assert!(next.is_none() || next.unwrap().is_err()); + } + + #[tokio::test] + async fn service_edits_close_only_affected_websockets() { + let f = fixture(); + let mut web = websocket(&f, "web").await; + let mut portal = SandboxServiceRoute { + endpoint_id: "portal-id".into(), + name: "portal".into(), + ..f.route.clone() + }; + f.routes + .install(&installed(vec![f.route.clone(), portal.clone()])); + echo(&mut web, "web survives portal addition").await; + let mut portal_ws = websocket(&f, "portal").await; + echo(&mut portal_ws, "portal active").await; + + // Renewing identical declarations does not revoke either stream. + f.routes + .install(&installed(vec![f.route.clone(), portal.clone()])); + echo(&mut web, "web renewed").await; + echo(&mut portal_ws, "portal renewed").await; + + portal.authorization_mode = 2; + portal.resource_version += 1; + f.routes + .install(&installed(vec![f.route.clone(), portal.clone()])); + closed(&mut portal_ws).await; + echo(&mut web, "web survives portal edit").await; + + let mut portal_ws = websocket(&f, "portal").await; + f.routes.install(&installed(vec![f.route.clone()])); + closed(&mut portal_ws).await; + echo(&mut web, "web survives portal deletion").await; + + f.routes.install(&installed(vec![f.route.clone(), portal])); + let mut portal_ws = websocket(&f, "portal").await; + f.routes.suspend(); + closed(&mut web).await; + closed(&mut portal_ws).await; + } + + #[tokio::test] + async fn coalesced_recreation_and_lease_lapse_cannot_revive_old_streams() { + let f = fixture(); + let (_, original) = f + .routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .unwrap(); + // Both updates can occur before a watch receiver observes either one. + f.routes.install(&installed(Vec::new())); + f.routes.install(&installed(vec![f.route.clone()])); + tokio::time::timeout(Duration::from_secs(1), f.routes.revoked(&f.route, original)) + .await + .unwrap(); + + let (_, renewed) = f + .routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .unwrap(); + f.routes.sender.send_modify(|snapshot| { + snapshot.expires = Some(Instant::now() - Duration::from_secs(1)); + }); + f.routes.install(&installed(vec![f.route.clone()])); + tokio::time::timeout(Duration::from_secs(1), f.routes.revoked(&f.route, renewed)) + .await + .unwrap(); + } +} diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index bdf4035c61..49d343b24e 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -825,6 +825,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { .store(true, Ordering::SeqCst); let startup = |await_main_process_attachment| { let config = crate::SandboxInstanceConfig { + app_router: None, command: vec!["test-agent".into(), "test-argument".into()], workdir: Some("/test-workspace".into()), timeout_secs: 15, @@ -1029,6 +1030,7 @@ async fn suspend_signal_exits_without_terminating_boundary() { let setup = TestSetup::new(); let startup = || { let config = crate::SandboxInstanceConfig { + app_router: None, command: vec!["test-agent".into(), "test-argument".into()], workdir: Some("/test-workspace".into()), timeout_secs: 15, diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index 8ced34ffec..504db41d26 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -904,6 +904,9 @@ impl ConfigRuntime { let requested_revision = sandbox_config_revision(&snapshot); let desired_environment = EnvironmentIdentity::from_settings(&snapshot); let provider = if !snapshot.configuration_admitted { + if let Some(routes) = &ctx.app_routes { + routes.suspend(); + } if !self.reloads_gateway_policy { return DeliveredResult { outcome: ConfigApplyOutcome::FailedRetainedLastKnownGood, @@ -938,6 +941,9 @@ impl ConfigRuntime { { Ok(prepared) => Some(prepared), Err(failure) => { + if let Some(routes) = &ctx.app_routes { + routes.suspend(); + } // Static credentials were revoked, so this configuration // is not applied until its environment installs. let outcome = if self.applied_revision.is_some() { @@ -968,6 +974,9 @@ impl ConfigRuntime { outcome, policy_status, }) => { + if let Some(routes) = &ctx.app_routes { + routes.install(&snapshot); + } self.stream.rejected_sandbox = None; let _ = ctx.workspace_tx.send(snapshot.workspace.clone()); let outcome = if !self.reloads_gateway_policy { @@ -987,6 +996,9 @@ impl ConfigRuntime { }; } Err(error) => { + if let Some(routes) = &ctx.app_routes { + routes.suspend(); + } // Polling ends its loop on this error; session delivery // reports it instead, so record it here. ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) @@ -1013,6 +1025,9 @@ impl ConfigRuntime { } }; + if let Some(routes) = &ctx.app_routes { + routes.suspend(); + } // The previous revision is retained only while its policy is enforced. let retained_revision = self .applied_revision @@ -1214,6 +1229,10 @@ async fn run_policy_poll_loop_with_client( client: C, ) -> Result<()> { let mut config_apply_rx = ctx.config_apply_rx.take(); + let _app_configuration = ctx + .app_routes + .clone() + .map(crate::app_listener::Routes::configuration_guard); let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); tokio::spawn(run_policy_status_reporter( client.clone(), @@ -1299,6 +1318,9 @@ async fn run_policy_poll_loop_with_client( ctx.provider_credentials.snapshot().revision, ) .await; + if let Some(routes) = &ctx.app_routes { + routes.install(&result); + } runtime.middleware_services = result.supervisor_middleware_services; runtime.extension_authentication_enabled = result.extension_authentication_enabled; @@ -1357,7 +1379,20 @@ async fn run_policy_poll_loop_with_client( // The session's bootstrap supersedes a polled result that was // not reconciled yet. pending_result = None; + // A quiet push stream still has authenticated session freshness. + // Renew only installed, unexpired routes. Poll mode renews by + // successfully applying a complete configuration instead. + if ctx.supervisor_session_id.borrow().is_some() + && let Some(routes) = &ctx.app_routes + { + routes.renew(); + } let delay = next_poll_delay(&ctx.extension_credentials, interval); + let delay = if ctx.app_routes.is_some() { + delay.min(Duration::from_secs(5)) + } else { + delay + }; tokio::select! { changed = async { match config_push_updates.as_mut() { @@ -1419,9 +1454,25 @@ async fn run_policy_poll_loop_with_client( } }; - let reconciled = runtime + let reconciled = match runtime .reconcile(&ctx, &client, &result, ProviderSource::Fetch) - .await?; + .await + { + Ok(reconciled) => reconciled, + Err(error) => { + if let Some(routes) = &ctx.app_routes { + routes.suspend(); + } + return Err(error); + } + }; + if let Some(routes) = &ctx.app_routes { + if matches!(reconciled.outcome, ReconcileOutcome::Rejected(_)) { + routes.suspend(); + } else { + routes.install(&result); + } + } if let Some(status) = reconciled.policy_status { enqueue_policy_status(&status_sender, status); } @@ -2722,6 +2773,7 @@ mod tests { ); } PolicyPollLoopContext { + app_routes: None, gateway: test_gateway(), sandbox_id: "sandbox-test".to_string(), sandbox: "sandbox-test-name".to_string(), diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index 2d3894d563..992bbbde18 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -278,6 +278,7 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl probe: probe.clone(), }; let config = SandboxInstanceConfig { + app_router: None, command: vec!["true".into()], workdir: None, timeout_secs: 0, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 8b1035beca..5f38715cb6 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -18,6 +18,7 @@ compile_error!( ); mod activity_aggregator; +pub mod app_listener; pub mod backend_setup; mod config_runtime; mod denial_aggregator; @@ -886,6 +887,7 @@ pub fn trace_context_from_environment() -> Option { /// from its [`SandboxInstanceBindings`], so one process can host several /// instances. pub struct SandboxInstanceConfig { + pub app_router: Option, pub command: Vec, pub workdir: Option, pub timeout_secs: u64, @@ -1051,6 +1053,7 @@ pub async fn run_sandbox_instance( controls: SandboxInstanceControls, ) -> Result { let SandboxInstanceConfig { + app_router, command, workdir, timeout_secs, @@ -1578,6 +1581,26 @@ pub async fn run_sandbox_instance( }); } + let app_instance = + match app_router { + Some(router) => { + if !loaded_policy_origin.allows_gateway_policy_reload() { + return Err(miette::miette!( + "app ingress requires gateway-managed configuration" + )); + } + Some( + router.instance(sandbox.as_deref().ok_or_else(|| { + miette::miette!("app ingress requires sandbox identity") + })?)?, + ) + } + None => None, + }; + let app_routes = app_instance + .as_ref() + .map(|instance| instance.routes.clone()); + // Spawn background policy poll task (gRPC mode only). if let (Some(id), Some(sandbox), Some(gateway), Some(engine)) = ( sandbox_id.as_deref(), @@ -1603,6 +1626,7 @@ pub async fn run_sandbox_instance( .and_then(|v| v.parse().ok()) .unwrap_or(10); let poll_ctx = PolicyPollLoopContext { + app_routes: app_routes.clone(), gateway: poll_gateway, sandbox_id: poll_id, sandbox: poll_sandbox, @@ -1688,6 +1712,8 @@ pub async fn run_sandbox_instance( } else { info!(backend = %backend_name, "Isolation boundary agent started"); } + let _app_listener = + app_instance.map(|instance| instance.start(running.loopback_connector())); let agent = running.agent(); let boundary_access = openshell_supervisor_process::delegated::start_boundary_access( sandbox_id.as_deref(), @@ -4193,6 +4219,7 @@ async fn report_initial_policy_failure( /// When the entrypoint PID is available, policy reloads include symlink /// resolution for binary paths via the container filesystem. struct PolicyPollLoopContext { + app_routes: Option, gateway: openshell_core::grpc_client::GatewayEndpoint, sandbox_id: String, /// Canonical sandbox reference used by name-scoped configuration APIs. @@ -4619,6 +4646,7 @@ mod tests { ocsf_otlp_headers: None, }; let config = SandboxInstanceConfig { + app_router: None, command: vec!["true".to_string()], workdir: None, timeout_secs: 0, @@ -5091,6 +5119,7 @@ network_policies: source: openshell_core::proto::PolicySource, ) -> openshell_core::grpc_client::SettingsPollResult { openshell_core::grpc_client::SettingsPollResult { + service_routes: Vec::new(), policy, version, policy_hash: format!("hash-v{version}"), @@ -7169,6 +7198,7 @@ network_policies: let canonical = settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); let canonical = openshell_core::grpc_client::SettingsPollResult { + service_routes: Vec::new(), policy_hash: "hash-provider-change".to_string(), config_revision: loaded.config_revision + 1, ..canonical @@ -7223,6 +7253,7 @@ network_policies: #[test] fn unchanged_sandbox_policy_revision_candidate_is_strictly_scoped() { let sandbox_result = openshell_core::grpc_client::SettingsPollResult { + service_routes: Vec::new(), policy_hash: "same-policy".to_string(), ..settings_poll_result( Some(proto_policy_fixture()), @@ -7263,6 +7294,7 @@ network_policies: ); let global_result = openshell_core::grpc_client::SettingsPollResult { + service_routes: Vec::new(), policy_hash: "same-policy".to_string(), ..settings_poll_result( Some(proto_policy_fixture()), diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 7c35b96826..ab94f199bb 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -597,7 +597,14 @@ fn main() -> Result<()> { "isolation-backend role started without validated runtime inputs" )); }; + let app_host = openshell_supervisor::app_listener::Config::from_env()? + .map(openshell_supervisor::app_listener::Config::bind) + .transpose()? + .map(openshell_supervisor::app_listener::BoundListener::start); let config = openshell_supervisor::SandboxInstanceConfig { + app_router: app_host + .as_ref() + .map(openshell_supervisor::app_listener::HostHandle::router), command, workdir, timeout_secs: args.timeout, diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs index 28f49b426b..305ebb3ca2 100644 --- a/crates/openshell-supervisor/src/shared_host.rs +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -182,6 +182,7 @@ pub struct SharedHostConfig { /// The sandbox instance registry and the routed runtime listener its /// instances claim routes on, reconciled to the gateway's assignments. pub struct SharedHost { + app_listener: Option, host: SandboxInstanceHost, router: Arc, gateway_endpoint: String, @@ -287,10 +288,15 @@ impl From for ControlError { impl SharedHost { /// Bind the routed runtime listener and prepare the instance root. pub fn start(config: SharedHostConfig) -> Result> { + let app_listener = crate::app_listener::Config::from_env()? + .map(crate::app_listener::Config::bind) + .transpose()? + .map(crate::app_listener::BoundListener::start); let host = SandboxInstanceHost::new(config.instance_root)?; let router = RuntimeRouter::bind(config.runtime_listen) .map_err(|error| miette::miette!("{error}"))?; Ok(Arc::new(Self { + app_listener, host, router, gateway_endpoint: config.gateway_endpoint, @@ -776,6 +782,10 @@ impl SharedHost { crate::sandbox_instance_host::prepare_private_dir(&trust_dir).map_err(failure)?; let ssh_socket = dir.join("ssh.sock"); let config = crate::SandboxInstanceConfig { + app_router: self + .app_listener + .as_ref() + .map(crate::app_listener::HostHandle::router), command: main_process.command, workdir: registration.workdir, timeout_secs: 0, diff --git a/deploy/helm/openshell/ci/values-app-listener.yaml b/deploy/helm/openshell/ci/values-app-listener.yaml new file mode 100644 index 0000000000..2604e1a6ff --- /dev/null +++ b/deploy/helm/openshell/ci/values-app-listener.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Opt-in fixture. Provision openshell-app-tls and an app-ingress NetworkPolicy +# before sending traffic. This does not deploy discovery or a proxy. +gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.supervisor.app_listener: + enabled: true + port: 9443 + domain: services.example.com + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + tls_secret_name: openshell-app-tls diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 48be86591d..73e4797b95 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -269,6 +269,11 @@ shared workspace mode. {{- $kubernetesConfig := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- $workspaceMode := get $kubernetesConfig "workspace_mode" | default "shared" -}} {{- $names := list -}} +{{- $app := include "openshell.appListenerConfig" . | fromYaml -}} +{{- $appDeployment := include "openshell.appListenerDeployment" . | fromYaml -}} +{{- if and (get $app "enabled") (get $appDeployment "tls_secret_name") (not (include "openshell.supervisorEnabled" .)) -}} +{{- $names = append $names (get $appDeployment "tls_secret_name") -}} +{{- end -}} {{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) -}} {{- $names = append $names .Values.server.tls.clientTlsSecretName -}} {{- end -}} @@ -631,6 +636,13 @@ absent fields so every chart consumer observes the same configuration. {{- end -}} {{- $legacyRuntime := .Values.supervisor.sandboxRuntime | default dict -}} {{- $runtimeConfig := get $kubernetes "sandbox_runtime" | default dict -}} +{{- if hasKey $gatewayConfig "openshell.drivers.kubernetes.sandbox_runtime" -}} +{{- $runtimeConfig = deepCopy (get $gatewayConfig "openshell.drivers.kubernetes.sandbox_runtime") -}} +{{- end -}} +{{- $appDeployment := include "openshell.appListenerDeployment" . | fromYaml -}} +{{- if $appDeployment -}} +{{- $_ := set $runtimeConfig "app_listener" $appDeployment -}} +{{- end -}} {{- if and (ne (int (get $legacyRuntime "boundaryPort" | default 5500)) 5500) (not (hasKey $runtimeConfig "boundary_port")) -}} {{- $_ := set $runtimeConfig "boundary_port" (int (get $legacyRuntime "boundaryPort")) -}} {{- end -}} @@ -807,3 +819,28 @@ Validate chart values that Helm would otherwise accept silently. {{- fail "server.dataStorageSize must be a Kubernetes quantity with a unit, such as 20Gi" -}} {{- end -}} {{- end }} + +{{/* Common app settings are separate from driver-owned TLS/Service details. */}} +{{- define "openshell.appListenerConfig" -}} +{{- $config := .Values.gatewayConfig | default dict -}} +{{- $supervisor := get $config "openshell.supervisor" | default dict -}} +{{- $app := get $supervisor "app_listener" | default dict -}} +{{- if hasKey $config "openshell.supervisor.app_listener" -}} +{{- $app = get $config "openshell.supervisor.app_listener" -}} +{{- end -}} +{{- toYaml $app -}} +{{- end -}} + +{{- define "openshell.appListenerDeployment" -}} +{{- $config := .Values.gatewayConfig | default dict -}} +{{- $kube := get $config "openshell.drivers.kubernetes" | default dict -}} +{{- $runtime := get $kube "sandbox_runtime" | default dict -}} +{{- if hasKey $config "openshell.drivers.kubernetes.sandbox_runtime" -}} +{{- $runtime = get $config "openshell.drivers.kubernetes.sandbox_runtime" -}} +{{- end -}} +{{- $app := get $runtime "app_listener" | default dict -}} +{{- if hasKey $config "openshell.drivers.kubernetes.sandbox_runtime.app_listener" -}} +{{- $app = get $config "openshell.drivers.kubernetes.sandbox_runtime.app_listener" -}} +{{- end -}} +{{- toYaml $app -}} +{{- end -}} diff --git a/deploy/helm/openshell/templates/_toml.tpl b/deploy/helm/openshell/templates/_toml.tpl index cc79d51d73..078c4c5f02 100644 --- a/deploy/helm/openshell/templates/_toml.tpl +++ b/deploy/helm/openshell/templates/_toml.tpl @@ -132,6 +132,9 @@ Top-level lists represent TOML arrays of tables and preserve their YAML order. * {{- $_ := set $config "openshell.gateway.gateway_jwt" $gatewayJwt -}} {{- $kubernetesCompat := include "openshell.effectiveKubernetesConfig" . | fromYaml -}} {{- $_ := set $config "openshell.drivers.kubernetes" $kubernetesCompat -}} +{{/* Runtime children are merged above: do not extend a sealed inline table. */}} +{{- $_ := unset $config "openshell.drivers.kubernetes.sandbox_runtime" -}} +{{- $_ := unset $config "openshell.drivers.kubernetes.sandbox_runtime.app_listener" -}} {{- $legacyDrivers := get $legacyServer "drivers" | default dict -}} {{- $legacyKubernetes := get $legacyDrivers "kubernetes" | default dict -}} {{- if hasKey $kubernetesCompat "resource_admission" -}} diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index 205803481d..513f49d283 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -9,6 +9,16 @@ # SPDX-License-Identifier: Apache-2.0 {{- $shared := get $kubernetes "shared_supervisor" }} {{- $name := include "openshell.supervisorName" . }} +{{- $app := include "openshell.appListenerConfig" . | fromYaml }} +{{- $appDeployment := include "openshell.appListenerDeployment" . | fromYaml }} +{{- $appPort := get $app "port" | default 8443 }} +{{- if get $app "enabled" }} +{{- $_ := required "supervisor app listener requires domain" (get $app "domain") }} +{{- $_ := required "supervisor app listener requires tls_secret_name" (get $appDeployment "tls_secret_name") }} +{{- if or (lt (int $appPort) 1024) (gt (int $appPort) 65535) (eq (int $appPort) 5501) (eq (int $appPort) (int (get $shared "runtime_port"))) }} +{{- fail "supervisor app listener port must be in 1024..65535 and distinct from runtime and health ports" }} +{{- end }} +{{- end }} {{- $endpoint := get $kubernetes "grpc_endpoint" }} {{- $endpointUrl := urlParse $endpoint }} {{- $gatewayTls := and (not .Values.server.disableTls) .Values.server.tls.clientTlsSecretName }} @@ -86,6 +96,18 @@ spec: - --session-root - /run/openshell/sessions env: + {{- if get $app "enabled" }} + - name: OPENSHELL_APP_LISTENER_ENABLED + value: "true" + - name: OPENSHELL_APP_LISTEN_ADDRESS + value: {{ printf "0.0.0.0:%v" $appPort | quote }} + - name: OPENSHELL_APP_SERVICE_DOMAIN + value: {{ get $app "domain" | quote }} + - name: OPENSHELL_APP_TLS_CERT + value: /etc/openshell/app-tls/tls.crt + - name: OPENSHELL_APP_TLS_KEY + value: /etc/openshell/app-tls/tls.key + {{- end }} - name: OPENSHELL_ENDPOINT value: {{ $endpoint | quote }} - name: OPENSHELL_LOG_LEVEL @@ -110,6 +132,11 @@ spec: fieldRef: fieldPath: status.podIP ports: + {{- if get $app "enabled" }} + - name: app + containerPort: {{ $appPort }} + protocol: TCP + {{- end }} - name: runtime containerPort: {{ get $shared "runtime_port" }} protocol: TCP @@ -137,6 +164,11 @@ spec: tcpSocket: port: health volumeMounts: + {{- if get $app "enabled" }} + - name: app-tls + mountPath: /etc/openshell/app-tls + readOnly: true + {{- end }} - name: run mountPath: /run/openshell - name: logs @@ -150,6 +182,11 @@ spec: readOnly: true {{- end }} volumes: + {{- if get $app "enabled" }} + - name: app-tls + secret: + secretName: {{ get $appDeployment "tls_secret_name" | quote }} + {{- end }} - name: run emptyDir: {} - name: logs diff --git a/deploy/helm/openshell/tests/app_listener_test.yaml b/deploy/helm/openshell/tests/app_listener_test.yaml new file mode 100644 index 0000000000..df59503b76 --- /dev/null +++ b/deploy/helm/openshell/tests/app_listener_test.yaml @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: supervisor app listener +templates: + - templates/supervisor.yaml +release: + name: openshell + namespace: gateway +tests: + - it: disabled configuration adds no socket port TLS mount or app Service + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.supervisor.app_listener: + enabled: false + port: 9443 + domain: services.example.com + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + tls_secret_name: app-tls + asserts: + - hasDocuments: + count: 1 + - lengthEqual: + path: spec.template.spec.containers[0].ports + count: 2 + documentIndex: 0 + - notContains: + path: spec.template.spec.volumes + content: + name: app-tls + any: true + documentIndex: 0 + - it: enables the common nondefault app port without creating a shared app Service + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.supervisor.app_listener: + enabled: true + port: 9443 + domain: services.example.com + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + tls_secret_name: app-tls + asserts: + - hasDocuments: + count: 1 + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_APP_LISTEN_ADDRESS + value: 0.0.0.0:9443 + documentIndex: 0 + - contains: + path: spec.template.spec.volumes + content: + name: app-tls + secret: + secretName: app-tls + documentIndex: 0 + - it: refuses app ports that collide with the runtime listener + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + openshell.supervisor.app_listener: + enabled: true + port: 5500 + domain: services.example.com + openshell.drivers.kubernetes.sandbox_runtime.app_listener: + tls_secret_name: app-tls + asserts: + - failedTemplate: + errorMessage: supervisor app listener port must be in 1024..65535 and distinct from runtime and health ports diff --git a/deploy/helm/test-gateway-config-parser.sh b/deploy/helm/test-gateway-config-parser.sh index 664af825df..6aeca1f9f0 100755 --- a/deploy/helm/test-gateway-config-parser.sh +++ b/deploy/helm/test-gateway-config-parser.sh @@ -43,6 +43,13 @@ render "${work_dir}/default.yaml" extract_toml "${work_dir}/default.yaml" "${work_dir}/default.toml" preflight "${work_dir}/default.toml" +# Explicit app ingress must survive both template rendering and driver +# preflight; child options cannot extend a sealed TOML inline table. +render "${work_dir}/app-listener.yaml" \ + --values "${chart}/ci/values-app-listener.yaml" +extract_toml "${work_dir}/app-listener.yaml" "${work_dir}/app-listener.toml" +preflight "${work_dir}/app-listener.toml" + # Exercise all serializer shapes through a raw driver table and prove that # literal strings, TOML escaping, and null omission survive parsing. render "${work_dir}/shapes.yaml" --values "${fixture}" diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 3d5680301d..420bb6b848 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1754,3 +1754,44 @@ Replace the executable and configuration paths with your installation's paths, a The command reports each selected executable path and version. Each tool receives only `-V`, with a five-second deadline and an 8 KiB output limit per stream. Missing, non-executable, outdated, or failing tools return a nonzero exit status with corrective guidance and the selected tool's error. The checks create no gateway state or images and do not start a driver or VM. Version checks confirm the installation; they do not test image creation or filesystem recovery. Other drivers do not require these VM tools. A VM configuration table alone does not select the VM driver. When a remote driver socket is configured, preflight reports that host tool checks were not performed; run the check on that host in the driver service's environment with a local VM configuration. + +## Supervisor application listener + +Enable a separate application TLS listener for declared sandbox HTTP and +WebSocket services. Application traffic reaches the supervisor directly and uses +authentication supplied by your application or ingress edge. Gateway service +ingress remains available independently. + +```toml +[openshell.supervisor.app_listener] +enabled = true +port = 9443 +domain = "services.example.com" + +[openshell.drivers.kubernetes.sandbox_runtime.app_listener] +tls_secret_name = "openshell-app-tls" +``` + +The listener defaults to disabled, with port `8443` and no domain. Port, domain, +or driver options alone do not enable it. Use an unprivileged port distinct from +the supervisor runtime and health ports. The common `port` controls both the +listener and its Kubernetes endpoint. + +On Kubernetes, provide a TLS Secret with `tls.crt` and `tls.key` matching +`*.services.example.com` in each supervisor namespace, and allow your ingress +clients to reach the app port through NetworkPolicy. The driver mounts the Secret +only in supervisor containers. Restart supervisors after changing certificates. +Dedicated and shared supervisors expose the app listener directly on their Pod +IPs. Route application traffic to the supervisor owning the sandbox, using the +configured app port. The driver creates no Kubernetes Service for application +traffic; ingress integrations manage any routing resources they need. +Existing supervisor processes retain their startup listener configuration until +restarted. + +Expose services with the existing commands. For direct listener testing, an +unnamed service on sandbox `demo` in workspace `default` uses SNI and Host +`default--demo.services.example.com`; a service named `web` uses +`default--demo--web.services.example.com`. Configure DNS or your ingress to reach +the owning supervisor while preserving that hostname. Automatic discovery and +application URL publication are separate work; current CLI URLs still identify +the existing gateway path. diff --git a/e2e/kubernetes/app-listener/README.md b/e2e/kubernetes/app-listener/README.md new file mode 100644 index 0000000000..3e26d81db6 --- /dev/null +++ b/e2e/kubernetes/app-listener/README.md @@ -0,0 +1,77 @@ +# Supervisor application listener verification + +Use a Kubernetes development cluster with the current gateway and supervisor +images. This fixture exercises the listener without discovery or the reference +proxy. The opt-in Helm values are +`deploy/helm/openshell/ci/values-app-listener.yaml`. + +Before deployment, provide an operator-owned `openshell-app-tls` Secret in the +supervisor namespace with `tls.crt` and `tls.key` for +`*.services.example.com`. Keep private keys outside the checkout. Allow a labeled +test-client Pod to reach supervisor TCP port 9443 through a separate +NetworkPolicy. The normal runtime/control port rules do not grant app access. +Deploy through the repository's Helm development workflow with these values. + +Create two sandboxes running an HTTP/WebSocket test application bound to loopback +port 8080. Expose named and unnamed services using the existing CLI, including +`openshell service expose 8080 web`. CLI URLs still use gateway mode in +this stage. Use these direct-listener hostnames instead: + +- `default--.services.example.com` for the unnamed service. +- `default----web.services.example.com` for the named service. + +Inspect supervisor Pods. Dedicated supervisors optionally expose +`os-app-`; inspect that Service's EndpointSlices. Shared supervisors +have no app Service: select the Pod IP actually owning the sandbox and use its +configured app port. Do not select an arbitrary member when more than one is +present. Check ownership through supervisor admission/release logs in this +stage; automated owner selection belongs to the later discovery API. + +From a permitted client Pod, mount only the public app certificate/CA and run: + +```shell +curl --cacert /test/app-ca.crt \ + --connect-to default--demo--web.services.example.com:443:SUPERVISOR_POD_IP:9443 \ + https://default--demo--web.services.example.com/ +``` + +For a dedicated supervisor, repeat using its headless Service DNS name instead +of the Pod IP. Verify both paths through ordinary cluster networking. A +port-forward alone does not verify Service selection or NetworkPolicy. + +Verify HTTP and WebSockets, public Host/Origin, application session cookies, +unknown SNI, mismatched Host, undeclared services, bearer stripping/passthrough, +and lack of access to another sandbox's ports. Change or remove one service +while its WebSocket is open: its connection closes while an unrelated service's +connection survives. Recreate the sandbox and confirm old streams cannot revive. + +Exercise configuration polling and push modes. In push mode, leave configuration +unchanged for longer than 30 seconds and confirm ingress remains usable while +the session is healthy. Reject configuration and confirm ingress closes. Stop +configuration/session refresh and require routes and streams to expire after +the remaining freshness lease once session loss is detected. + +Run with listener configuration omitted, explicitly disabled despite complete +TLS/port options, and explicitly enabled. Disabled deployments have no app port, +TLS mount, or app Service. Retain the dedicated driver `create_service` option +while disabling and recreating a sandbox to verify cleanup of its previously +owned endpoint. Confirm gateway ingress still works in each mode. + +Dedicated endpoint reconciliation runs more than once during creation. Verify +that it accepts its own non-controller Sandbox owner reference and has Service +`patch`/`delete` permissions when `create_service` is configured. Foreign-owned +Services must be left untouched, and cleanup uses UID/resource-version +preconditions. + +For HA pool validation after integration with #4864, use at least two supervisor +members with distinct sandbox owners. Move one owner through graceful downscale +and abrupt loss. Existing streams may disconnect; after selecting the new owner, +new HTTP/WebSocket connections must work without restarting the workload. +Validate unchanged workload generation/process memory, rejection of old-owner +connectors, and continued service for the unaffected sandbox. A static ingress +must be updated manually in this stage; automated owner tracking belongs to +subsequent discovery work. + +The local Rust listener tests use a real TLS client and backend RPC connector. +They supplement this live cluster check; they do not establish HA pool recovery +or real Kubernetes reachability. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 80344bd4a1..b57c5d4cda 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -427,6 +427,8 @@ message SandboxConfigSnapshot { string configuration_instance_id = 15; string configuration_error = 16; uint64 settings_revision = 17; + // Explicit application ingress allowlist. Absent means no app routes. + repeated SandboxServiceRoute service_routes = 18; } // Response containing effective sandbox settings and policy, projected from @@ -477,6 +479,17 @@ message GetSandboxConfigResponse { string configuration_instance_id = 15; // Monotonic revision of the sandbox-scoped settings row. uint64 settings_revision = 17; + repeated SandboxServiceRoute service_routes = 18; +} + +// Gateway-declared service installed by a supervisor; this does not grant +// access to arbitrary workload ports or assert application health. +message SandboxServiceRoute { + string endpoint_id = 1; + string name = 2; + uint32 target_port = 3; + int32 authorization_mode = 4; + uint64 resource_version = 5; } // Connection details for one operator-registered supervisor middleware service. diff --git a/sdk/go/proto/sandboxv1/sandbox.pb.go b/sdk/go/proto/sandboxv1/sandbox.pb.go index 4b09b2016c..045e15fac5 100644 --- a/sdk/go/proto/sandboxv1/sandbox.pb.go +++ b/sdk/go/proto/sandboxv1/sandbox.pb.go @@ -2116,8 +2116,10 @@ type SandboxConfigSnapshot struct { ConfigurationInstanceId string `protobuf:"bytes,15,opt,name=configuration_instance_id,json=configurationInstanceId,proto3" json:"configuration_instance_id,omitempty"` ConfigurationError string `protobuf:"bytes,16,opt,name=configuration_error,json=configurationError,proto3" json:"configuration_error,omitempty"` SettingsRevision uint64 `protobuf:"varint,17,opt,name=settings_revision,json=settingsRevision,proto3" json:"settings_revision,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Explicit application ingress allowlist. Absent means no app routes. + ServiceRoutes []*SandboxServiceRoute `protobuf:"bytes,18,rep,name=service_routes,json=serviceRoutes,proto3" json:"service_routes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SandboxConfigSnapshot) Reset() { @@ -2269,6 +2271,13 @@ func (x *SandboxConfigSnapshot) GetSettingsRevision() uint64 { return 0 } +func (x *SandboxConfigSnapshot) GetServiceRoutes() []*SandboxServiceRoute { + if x != nil { + return x.ServiceRoutes + } + return nil +} + // Response containing effective sandbox settings and policy, projected from // the SandboxConfigSnapshot that push supervisors receive. type GetSandboxConfigResponse struct { @@ -2317,7 +2326,8 @@ type GetSandboxConfigResponse struct { // Registration fence for a new supervisor; capture once and retain on retry. ConfigurationInstanceId string `protobuf:"bytes,15,opt,name=configuration_instance_id,json=configurationInstanceId,proto3" json:"configuration_instance_id,omitempty"` // Monotonic revision of the sandbox-scoped settings row. - SettingsRevision uint64 `protobuf:"varint,17,opt,name=settings_revision,json=settingsRevision,proto3" json:"settings_revision,omitempty"` + SettingsRevision uint64 `protobuf:"varint,17,opt,name=settings_revision,json=settingsRevision,proto3" json:"settings_revision,omitempty"` + ServiceRoutes []*SandboxServiceRoute `protobuf:"bytes,18,rep,name=service_routes,json=serviceRoutes,proto3" json:"service_routes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2471,6 +2481,91 @@ func (x *GetSandboxConfigResponse) GetSettingsRevision() uint64 { return 0 } +func (x *GetSandboxConfigResponse) GetServiceRoutes() []*SandboxServiceRoute { + if x != nil { + return x.ServiceRoutes + } + return nil +} + +// Gateway-declared service installed by a supervisor; this does not grant +// access to arbitrary workload ports or assert application health. +type SandboxServiceRoute struct { + state protoimpl.MessageState `protogen:"open.v1"` + EndpointId string `protobuf:"bytes,1,opt,name=endpoint_id,json=endpointId,proto3" json:"endpoint_id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + TargetPort uint32 `protobuf:"varint,3,opt,name=target_port,json=targetPort,proto3" json:"target_port,omitempty"` + AuthorizationMode int32 `protobuf:"varint,4,opt,name=authorization_mode,json=authorizationMode,proto3" json:"authorization_mode,omitempty"` + ResourceVersion uint64 `protobuf:"varint,5,opt,name=resource_version,json=resourceVersion,proto3" json:"resource_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SandboxServiceRoute) Reset() { + *x = SandboxServiceRoute{} + mi := &file_sandbox_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SandboxServiceRoute) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SandboxServiceRoute) ProtoMessage() {} + +func (x *SandboxServiceRoute) ProtoReflect() protoreflect.Message { + mi := &file_sandbox_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SandboxServiceRoute.ProtoReflect.Descriptor instead. +func (*SandboxServiceRoute) Descriptor() ([]byte, []int) { + return file_sandbox_proto_rawDescGZIP(), []int{23} +} + +func (x *SandboxServiceRoute) GetEndpointId() string { + if x != nil { + return x.EndpointId + } + return "" +} + +func (x *SandboxServiceRoute) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SandboxServiceRoute) GetTargetPort() uint32 { + if x != nil { + return x.TargetPort + } + return 0 +} + +func (x *SandboxServiceRoute) GetAuthorizationMode() int32 { + if x != nil { + return x.AuthorizationMode + } + return 0 +} + +func (x *SandboxServiceRoute) GetResourceVersion() uint64 { + if x != nil { + return x.ResourceVersion + } + return 0 +} + // Connection details for one operator-registered supervisor middleware service. // V1 supports plaintext and server-authenticated TLS gRPC. type SupervisorMiddlewareService struct { @@ -2510,7 +2605,7 @@ type SupervisorMiddlewareService struct { func (x *SupervisorMiddlewareService) Reset() { *x = SupervisorMiddlewareService{} - mi := &file_sandbox_proto_msgTypes[23] + mi := &file_sandbox_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2522,7 +2617,7 @@ func (x *SupervisorMiddlewareService) String() string { func (*SupervisorMiddlewareService) ProtoMessage() {} func (x *SupervisorMiddlewareService) ProtoReflect() protoreflect.Message { - mi := &file_sandbox_proto_msgTypes[23] + mi := &file_sandbox_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2535,7 +2630,7 @@ func (x *SupervisorMiddlewareService) ProtoReflect() protoreflect.Message { // Deprecated: Use SupervisorMiddlewareService.ProtoReflect.Descriptor instead. func (*SupervisorMiddlewareService) Descriptor() ([]byte, []int) { - return file_sandbox_proto_rawDescGZIP(), []int{23} + return file_sandbox_proto_rawDescGZIP(), []int{24} } func (x *SupervisorMiddlewareService) GetName() string { @@ -2755,7 +2850,7 @@ const file_sandbox_proto_rawDesc = "" + "\x05value\"\x86\x01\n" + "\x10EffectiveSetting\x128\n" + "\x05value\x18\x01 \x01(\v2\".openshell.sandbox.v1.SettingValueR\x05value\x128\n" + - "\x05scope\x18\x02 \x01(\x0e2\".openshell.sandbox.v1.SettingScopeR\x05scope\"\xd8\b\n" + + "\x05scope\x18\x02 \x01(\x0e2\".openshell.sandbox.v1.SettingScopeR\x05scope\"\xaa\t\n" + "\x15SandboxConfigSnapshot\x12;\n" + "\x06policy\x18\x01 \x01(\v2#.openshell.sandbox.v1.SandboxPolicyR\x06policy\x12\x18\n" + "\aversion\x18\x02 \x01(\rR\aversion\x12\x1f\n" + @@ -2775,10 +2870,11 @@ const file_sandbox_proto_rawDesc = "" + "\x19provider_attachment_epoch\x18\x0e \x01(\tR\x17providerAttachmentEpoch\x12:\n" + "\x19configuration_instance_id\x18\x0f \x01(\tR\x17configurationInstanceId\x12/\n" + "\x13configuration_error\x18\x10 \x01(\tR\x12configurationError\x12+\n" + - "\x11settings_revision\x18\x11 \x01(\x04R\x10settingsRevision\x1ac\n" + + "\x11settings_revision\x18\x11 \x01(\x04R\x10settingsRevision\x12P\n" + + "\x0eservice_routes\x18\x12 \x03(\v2).openshell.sandbox.v1.SandboxServiceRouteR\rserviceRoutes\x1ac\n" + "\rSettingsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12<\n" + - "\x05value\x18\x02 \x01(\v2&.openshell.sandbox.v1.EffectiveSettingR\x05value:\x028\x01\"\xde\b\n" + + "\x05value\x18\x02 \x01(\v2&.openshell.sandbox.v1.EffectiveSettingR\x05value:\x028\x01\"\xb0\t\n" + "\x18GetSandboxConfigResponse\x12;\n" + "\x06policy\x18\x01 \x01(\v2#.openshell.sandbox.v1.SandboxPolicyR\x06policy\x12\x18\n" + "\aversion\x18\x02 \x01(\rR\aversion\x12\x1f\n" + @@ -2798,10 +2894,19 @@ const file_sandbox_proto_rawDesc = "" + "\x16configuration_admitted\x18\r \x01(\bR\x15configurationAdmitted\x12/\n" + "\x13configuration_error\x18\x10 \x01(\tR\x12configurationError\x12:\n" + "\x19configuration_instance_id\x18\x0f \x01(\tR\x17configurationInstanceId\x12+\n" + - "\x11settings_revision\x18\x11 \x01(\x04R\x10settingsRevision\x1ac\n" + + "\x11settings_revision\x18\x11 \x01(\x04R\x10settingsRevision\x12P\n" + + "\x0eservice_routes\x18\x12 \x03(\v2).openshell.sandbox.v1.SandboxServiceRouteR\rserviceRoutes\x1ac\n" + "\rSettingsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12<\n" + - "\x05value\x18\x02 \x01(\v2&.openshell.sandbox.v1.EffectiveSettingR\x05value:\x028\x01\"\x81\x04\n" + + "\x05value\x18\x02 \x01(\v2&.openshell.sandbox.v1.EffectiveSettingR\x05value:\x028\x01\"\xc5\x01\n" + + "\x13SandboxServiceRoute\x12\x1f\n" + + "\vendpoint_id\x18\x01 \x01(\tR\n" + + "endpointId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x1f\n" + + "\vtarget_port\x18\x03 \x01(\rR\n" + + "targetPort\x12-\n" + + "\x12authorization_mode\x18\x04 \x01(\x05R\x11authorizationMode\x12)\n" + + "\x10resource_version\x18\x05 \x01(\x04R\x0fresourceVersion\"\x81\x04\n" + "\x1bSupervisorMiddlewareService\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12#\n" + "\rgrpc_endpoint\x18\x02 \x01(\tR\fgrpcEndpoint\x12*\n" + @@ -2858,7 +2963,7 @@ func file_sandbox_proto_rawDescGZIP() []byte { } var file_sandbox_proto_enumTypes = make([]protoimpl.EnumInfo, 7) -var file_sandbox_proto_msgTypes = make([]protoimpl.MessageInfo, 34) +var file_sandbox_proto_msgTypes = make([]protoimpl.MessageInfo, 35) var file_sandbox_proto_goTypes = []any{ (NetworkTlsMode)(0), // 0: openshell.sandbox.v1.NetworkTlsMode (NetworkEnforcementMode)(0), // 1: openshell.sandbox.v1.NetworkEnforcementMode @@ -2890,74 +2995,77 @@ var file_sandbox_proto_goTypes = []any{ (*EffectiveSetting)(nil), // 27: openshell.sandbox.v1.EffectiveSetting (*SandboxConfigSnapshot)(nil), // 28: openshell.sandbox.v1.SandboxConfigSnapshot (*GetSandboxConfigResponse)(nil), // 29: openshell.sandbox.v1.GetSandboxConfigResponse - (*SupervisorMiddlewareService)(nil), // 30: openshell.sandbox.v1.SupervisorMiddlewareService - nil, // 31: openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry - nil, // 32: openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry - nil, // 33: openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry - nil, // 34: openshell.sandbox.v1.L7DenyRule.QueryEntry - nil, // 35: openshell.sandbox.v1.L7DenyRule.ParamsEntry - nil, // 36: openshell.sandbox.v1.L7Allow.QueryEntry - nil, // 37: openshell.sandbox.v1.L7Allow.ParamsEntry - nil, // 38: openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry - nil, // 39: openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry - nil, // 40: openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry - (*structpb.Struct)(nil), // 41: google.protobuf.Struct - (*datamodelv1.WorkspaceSelector)(nil), // 42: openshell.datamodel.v1.WorkspaceSelector - (*durationpb.Duration)(nil), // 43: google.protobuf.Duration + (*SandboxServiceRoute)(nil), // 30: openshell.sandbox.v1.SandboxServiceRoute + (*SupervisorMiddlewareService)(nil), // 31: openshell.sandbox.v1.SupervisorMiddlewareService + nil, // 32: openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry + nil, // 33: openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry + nil, // 34: openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry + nil, // 35: openshell.sandbox.v1.L7DenyRule.QueryEntry + nil, // 36: openshell.sandbox.v1.L7DenyRule.ParamsEntry + nil, // 37: openshell.sandbox.v1.L7Allow.QueryEntry + nil, // 38: openshell.sandbox.v1.L7Allow.ParamsEntry + nil, // 39: openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry + nil, // 40: openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry + nil, // 41: openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry + (*structpb.Struct)(nil), // 42: google.protobuf.Struct + (*datamodelv1.WorkspaceSelector)(nil), // 43: openshell.datamodel.v1.WorkspaceSelector + (*durationpb.Duration)(nil), // 44: google.protobuf.Duration } var file_sandbox_proto_depIdxs = []int32{ 8, // 0: openshell.sandbox.v1.SandboxPolicy.filesystem:type_name -> openshell.sandbox.v1.FilesystemPolicy 9, // 1: openshell.sandbox.v1.SandboxPolicy.landlock:type_name -> openshell.sandbox.v1.LandlockPolicy 10, // 2: openshell.sandbox.v1.SandboxPolicy.process:type_name -> openshell.sandbox.v1.ProcessPolicy - 31, // 3: openshell.sandbox.v1.SandboxPolicy.network_policies:type_name -> openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry - 32, // 4: openshell.sandbox.v1.SandboxPolicy.network_middlewares:type_name -> openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry + 32, // 3: openshell.sandbox.v1.SandboxPolicy.network_policies:type_name -> openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry + 33, // 4: openshell.sandbox.v1.SandboxPolicy.network_middlewares:type_name -> openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry 15, // 5: openshell.sandbox.v1.NetworkPolicyRule.endpoints:type_name -> openshell.sandbox.v1.NetworkEndpoint 22, // 6: openshell.sandbox.v1.NetworkPolicyRule.binaries:type_name -> openshell.sandbox.v1.NetworkBinary - 41, // 7: openshell.sandbox.v1.NetworkMiddlewareConfig.config:type_name -> google.protobuf.Struct + 42, // 7: openshell.sandbox.v1.NetworkMiddlewareConfig.config:type_name -> google.protobuf.Struct 13, // 8: openshell.sandbox.v1.NetworkMiddlewareConfig.endpoints:type_name -> openshell.sandbox.v1.MiddlewareEndpointSelector 0, // 9: openshell.sandbox.v1.NetworkEndpoint.tls:type_name -> openshell.sandbox.v1.NetworkTlsMode 1, // 10: openshell.sandbox.v1.NetworkEndpoint.enforcement:type_name -> openshell.sandbox.v1.NetworkEnforcementMode 2, // 11: openshell.sandbox.v1.NetworkEndpoint.access:type_name -> openshell.sandbox.v1.NetworkAccessPreset 19, // 12: openshell.sandbox.v1.NetworkEndpoint.rules:type_name -> openshell.sandbox.v1.L7Rule 18, // 13: openshell.sandbox.v1.NetworkEndpoint.deny_rules:type_name -> openshell.sandbox.v1.L7DenyRule - 33, // 14: openshell.sandbox.v1.NetworkEndpoint.graphql_persisted_queries:type_name -> openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry + 34, // 14: openshell.sandbox.v1.NetworkEndpoint.graphql_persisted_queries:type_name -> openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry 16, // 15: openshell.sandbox.v1.NetworkEndpoint.mcp:type_name -> openshell.sandbox.v1.McpOptions 14, // 16: openshell.sandbox.v1.NetworkEndpoint.credential_binding:type_name -> openshell.sandbox.v1.NetworkCredentialBinding - 34, // 17: openshell.sandbox.v1.L7DenyRule.query:type_name -> openshell.sandbox.v1.L7DenyRule.QueryEntry - 35, // 18: openshell.sandbox.v1.L7DenyRule.params:type_name -> openshell.sandbox.v1.L7DenyRule.ParamsEntry + 35, // 17: openshell.sandbox.v1.L7DenyRule.query:type_name -> openshell.sandbox.v1.L7DenyRule.QueryEntry + 36, // 18: openshell.sandbox.v1.L7DenyRule.params:type_name -> openshell.sandbox.v1.L7DenyRule.ParamsEntry 20, // 19: openshell.sandbox.v1.L7Rule.allow:type_name -> openshell.sandbox.v1.L7Allow - 36, // 20: openshell.sandbox.v1.L7Allow.query:type_name -> openshell.sandbox.v1.L7Allow.QueryEntry - 37, // 21: openshell.sandbox.v1.L7Allow.params:type_name -> openshell.sandbox.v1.L7Allow.ParamsEntry - 42, // 22: openshell.sandbox.v1.GetSandboxConfigRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 38, // 23: openshell.sandbox.v1.GetGatewayConfigResponse.settings:type_name -> openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry + 37, // 20: openshell.sandbox.v1.L7Allow.query:type_name -> openshell.sandbox.v1.L7Allow.QueryEntry + 38, // 21: openshell.sandbox.v1.L7Allow.params:type_name -> openshell.sandbox.v1.L7Allow.ParamsEntry + 43, // 22: openshell.sandbox.v1.GetSandboxConfigRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 39, // 23: openshell.sandbox.v1.GetGatewayConfigResponse.settings:type_name -> openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry 26, // 24: openshell.sandbox.v1.EffectiveSetting.value:type_name -> openshell.sandbox.v1.SettingValue 3, // 25: openshell.sandbox.v1.EffectiveSetting.scope:type_name -> openshell.sandbox.v1.SettingScope 7, // 26: openshell.sandbox.v1.SandboxConfigSnapshot.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 39, // 27: openshell.sandbox.v1.SandboxConfigSnapshot.settings:type_name -> openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry + 40, // 27: openshell.sandbox.v1.SandboxConfigSnapshot.settings:type_name -> openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry 4, // 28: openshell.sandbox.v1.SandboxConfigSnapshot.policy_source:type_name -> openshell.sandbox.v1.PolicySource - 30, // 29: openshell.sandbox.v1.SandboxConfigSnapshot.supervisor_middleware_services:type_name -> openshell.sandbox.v1.SupervisorMiddlewareService - 7, // 30: openshell.sandbox.v1.GetSandboxConfigResponse.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 40, // 31: openshell.sandbox.v1.GetSandboxConfigResponse.settings:type_name -> openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry - 4, // 32: openshell.sandbox.v1.GetSandboxConfigResponse.policy_source:type_name -> openshell.sandbox.v1.PolicySource - 30, // 33: openshell.sandbox.v1.GetSandboxConfigResponse.supervisor_middleware_services:type_name -> openshell.sandbox.v1.SupervisorMiddlewareService - 43, // 34: openshell.sandbox.v1.SupervisorMiddlewareService.request_timeout:type_name -> google.protobuf.Duration - 5, // 35: openshell.sandbox.v1.SupervisorMiddlewareService.binding_policy:type_name -> openshell.sandbox.v1.SupervisorMiddlewareBindingPolicy - 6, // 36: openshell.sandbox.v1.SupervisorMiddlewareService.bindings:type_name -> openshell.sandbox.v1.SupervisorMiddlewareBinding - 11, // 37: openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry.value:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 12, // 38: openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry.value:type_name -> openshell.sandbox.v1.NetworkMiddlewareConfig - 17, // 39: openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry.value:type_name -> openshell.sandbox.v1.GraphqlOperation - 21, // 40: openshell.sandbox.v1.L7DenyRule.QueryEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher - 21, // 41: openshell.sandbox.v1.L7DenyRule.ParamsEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher - 21, // 42: openshell.sandbox.v1.L7Allow.QueryEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher - 21, // 43: openshell.sandbox.v1.L7Allow.ParamsEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher - 26, // 44: openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry.value:type_name -> openshell.sandbox.v1.SettingValue - 27, // 45: openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry.value:type_name -> openshell.sandbox.v1.EffectiveSetting - 27, // 46: openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry.value:type_name -> openshell.sandbox.v1.EffectiveSetting - 47, // [47:47] is the sub-list for method output_type - 47, // [47:47] is the sub-list for method input_type - 47, // [47:47] is the sub-list for extension type_name - 47, // [47:47] is the sub-list for extension extendee - 0, // [0:47] is the sub-list for field type_name + 31, // 29: openshell.sandbox.v1.SandboxConfigSnapshot.supervisor_middleware_services:type_name -> openshell.sandbox.v1.SupervisorMiddlewareService + 30, // 30: openshell.sandbox.v1.SandboxConfigSnapshot.service_routes:type_name -> openshell.sandbox.v1.SandboxServiceRoute + 7, // 31: openshell.sandbox.v1.GetSandboxConfigResponse.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 41, // 32: openshell.sandbox.v1.GetSandboxConfigResponse.settings:type_name -> openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry + 4, // 33: openshell.sandbox.v1.GetSandboxConfigResponse.policy_source:type_name -> openshell.sandbox.v1.PolicySource + 31, // 34: openshell.sandbox.v1.GetSandboxConfigResponse.supervisor_middleware_services:type_name -> openshell.sandbox.v1.SupervisorMiddlewareService + 30, // 35: openshell.sandbox.v1.GetSandboxConfigResponse.service_routes:type_name -> openshell.sandbox.v1.SandboxServiceRoute + 44, // 36: openshell.sandbox.v1.SupervisorMiddlewareService.request_timeout:type_name -> google.protobuf.Duration + 5, // 37: openshell.sandbox.v1.SupervisorMiddlewareService.binding_policy:type_name -> openshell.sandbox.v1.SupervisorMiddlewareBindingPolicy + 6, // 38: openshell.sandbox.v1.SupervisorMiddlewareService.bindings:type_name -> openshell.sandbox.v1.SupervisorMiddlewareBinding + 11, // 39: openshell.sandbox.v1.SandboxPolicy.NetworkPoliciesEntry.value:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 12, // 40: openshell.sandbox.v1.SandboxPolicy.NetworkMiddlewaresEntry.value:type_name -> openshell.sandbox.v1.NetworkMiddlewareConfig + 17, // 41: openshell.sandbox.v1.NetworkEndpoint.GraphqlPersistedQueriesEntry.value:type_name -> openshell.sandbox.v1.GraphqlOperation + 21, // 42: openshell.sandbox.v1.L7DenyRule.QueryEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher + 21, // 43: openshell.sandbox.v1.L7DenyRule.ParamsEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher + 21, // 44: openshell.sandbox.v1.L7Allow.QueryEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher + 21, // 45: openshell.sandbox.v1.L7Allow.ParamsEntry.value:type_name -> openshell.sandbox.v1.L7QueryMatcher + 26, // 46: openshell.sandbox.v1.GetGatewayConfigResponse.SettingsEntry.value:type_name -> openshell.sandbox.v1.SettingValue + 27, // 47: openshell.sandbox.v1.SandboxConfigSnapshot.SettingsEntry.value:type_name -> openshell.sandbox.v1.EffectiveSetting + 27, // 48: openshell.sandbox.v1.GetSandboxConfigResponse.SettingsEntry.value:type_name -> openshell.sandbox.v1.EffectiveSetting + 49, // [49:49] is the sub-list for method output_type + 49, // [49:49] is the sub-list for method input_type + 49, // [49:49] is the sub-list for extension type_name + 49, // [49:49] is the sub-list for extension extendee + 0, // [0:49] is the sub-list for field type_name } func init() { file_sandbox_proto_init() } @@ -2978,7 +3086,7 @@ func file_sandbox_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_sandbox_proto_rawDesc), len(file_sandbox_proto_rawDesc)), NumEnums: 7, - NumMessages: 34, + NumMessages: 35, NumExtensions: 0, NumServices: 0, }, diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 9e56f7117c..879ab84d91 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1622,3 +1622,27 @@ probes. Arguments after `--` validate the effective daemon invocation, including its command-line overrides. Preflight preserves every failed file. Do not advise users to delete or rewrite it automatically; back it up and follow the manual schema-v2 migration in the Gateway Configuration reference. + +## Supervisor application ingress + +When `[openshell.supervisor.app_listener]` is explicitly enabled, check its +`port` and `domain` separately from the gateway API and supervisor runtime ports. +The Kubernetes TLS Secret is configured with `tls_secret_name` under +`[openshell.drivers.kubernetes.sandbox_runtime.app_listener]`. Disabled listeners +create no app TLS mount or app port. TLS material is mounted only in supervisors +and reloads on supervisor restart. + +Inspect supervisor Pods with `kubectl`. In both dedicated and shared topologies, +connect to the current sandbox owner's Pod IP and configured app port. The driver +creates no app Services. Preserve service SNI and Host when using curl's +`--connect-to`, and verify the application certificate. A port-forward bypasses +NetworkPolicy forwarding; also test from a permitted in-cluster client. Operators +provide ingress network rules. Running supervisors retain their startup listener +configuration until restarted. + +Service changes arrive through polling or push configuration. Only admitted +service declarations are installed. Polling and accepted live push sessions +renew a 30-second lease; rejected configuration or expired freshness closes +routes. App traffic uses application/edge authentication independently of gateway +user authentication. This listener stage does not add service discovery or +change the public URLs printed by the CLI. From a1484aa54c45805fd21dcd4a742bd3271048adb4 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 28 Oct 2026 15:51:50 -0700 Subject: [PATCH 2/5] test(supervisor): automate Kubernetes application listener e2e Signed-off-by: Taylor Mutch --- .agents/skills/helm-dev-environment/SKILL.md | 14 +- .github/actions/setup-e2e-kind/action.yml | 39 +- .github/workflows/branch-e2e.yml | 38 ++ .github/workflows/e2e-kubernetes-test.yml | 36 + e2e/kubernetes/app-listener/README.md | 124 +++- e2e/kubernetes/app-listener/app.py | 70 ++ e2e/kubernetes/app-listener/client.py | 184 +++++ e2e/kubernetes/app-listener/run.py | 367 ++++++++++ e2e/kubernetes/app-listener/verify.py | 683 +++++++++++++++++++ mise.toml | 2 +- tasks/test.toml | 4 + 11 files changed, 1538 insertions(+), 23 deletions(-) create mode 100644 e2e/kubernetes/app-listener/app.py create mode 100644 e2e/kubernetes/app-listener/client.py create mode 100644 e2e/kubernetes/app-listener/run.py create mode 100644 e2e/kubernetes/app-listener/verify.py diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index d3d4cdfb7f..a977e722d6 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -445,6 +445,16 @@ public `SandboxStatus` image fields. Resume a pending operation with `SandboxUpdates.UpdateSandbox`, the current resource version, and no selected fields, following the documented request-ID semantics. + +For the opt-in application listener, build/import matching branch images and run +`mise run e2e:kubernetes:app-listener`; see `e2e/kubernetes/app-listener/README.md` +for image, topology, delivery-mode and takeover inputs. The task owns an isolated +Helm release/namespace, temporary TLS material, in-cluster clients, teardown and +evidence under `results/app-listener/`. It checks policy denial, Pod-IP ingress without app Services, shared-owner takeover, gateway-outage expiry and disabled-listener +gateway compatibility, without discovery or a proxy. CI runs four cases under +`test:e2e` and includes them in `Core E2E result`. Its reusable kind setup installs +pinned Calico only for this lane; other jobs retain kindnet. Local k3s already +enforces policy. The lower-level `verify.py` can target a predeployed release. The kube e2e wrapper creates only one port-forward, to `svc/openshell`; it no longer forwards the unauthenticated health listener or runs a `/readyz` e2e target. `/readyz` remains covered by server unit/integration tests. @@ -653,6 +663,6 @@ supervisor listener at port 9443 without discovery or a proxy. Provision the operator TLS Secret and an ingress-client NetworkPolicy first. Follow `e2e/kubernetes/app-listener/README.md` for direct in-cluster verification, shared-member selection, and the HA follow-up checks. Keep the fixture disabled -in default profiles. Shared supervisors expose app ports on their Pod IPs -without a shared app Service. Select the member owning the sandbox; automatic +in default profiles. Dedicated and shared supervisors expose app ports on their +Pod IPs without app Services. Select the member owning the sandbox; automatic owner selection belongs to the later discovery API. diff --git a/.github/actions/setup-e2e-kind/action.yml b/.github/actions/setup-e2e-kind/action.yml index e617a27717..285f0eb145 100644 --- a/.github/actions/setup-e2e-kind/action.yml +++ b/.github/actions/setup-e2e-kind/action.yml @@ -29,6 +29,10 @@ inputs: description: mise release to install required: false default: v2026.9.9 + enforce-network-policy: + description: Replace kindnet with pinned Calico for NetworkPolicy assertions + required: false + default: "false" runs: using: composite @@ -45,11 +49,44 @@ runs: REGISTRY_PASSWORD: ${{ inputs.registry-password }} run: echo "$REGISTRY_PASSWORD" | docker login "${REGISTRY%%/*}" -u "$REGISTRY_USERNAME" --password-stdin + - name: Configure policy-enforcing kind network + id: kind-config + if: inputs.enforce-network-policy == 'true' + shell: bash + run: | + set -euo pipefail + config="$RUNNER_TEMP/kind-network-policy.yaml" + cat > "$config" <<'YAML' + kind: Cluster + apiVersion: kind.x-k8s.io/v1alpha4 + networking: + disableDefaultCNI: true + podSubnet: 10.244.0.0/16 + YAML + echo "path=$config" >> "$GITHUB_OUTPUT" + - name: Create kind cluster uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 with: cluster_name: ${{ inputs.cluster-name }} - wait: 120s + # Nodes cannot become Ready until the following CNI installation. + wait: ${{ inputs.enforce-network-policy == 'true' && '0s' || '120s' }} + config: ${{ steps.kind-config.outputs.path }} + + - name: Install policy-enforcing CNI + if: inputs.enforce-network-policy == 'true' + shell: bash + env: + CLUSTER_NAME: ${{ inputs.cluster-name }} + run: | + set -euo pipefail + curl -fsSL https://raw.githubusercontent.com/projectcalico/calico/v3.30.3/manifests/calico.yaml \ + -o "$RUNNER_TEMP/calico.yaml" + sed 's@192.168.0.0/16@10.244.0.0/16@g' "$RUNNER_TEMP/calico.yaml" > "$RUNNER_TEMP/calico-kind.yaml" + kubectl --context "kind-$CLUSTER_NAME" apply -f "$RUNNER_TEMP/calico-kind.yaml" + kubectl --context "kind-$CLUSTER_NAME" -n kube-system rollout status daemonset/calico-node --timeout=300s + kubectl --context "kind-$CLUSTER_NAME" -n kube-system rollout status deployment/calico-kube-controllers --timeout=300s + kubectl --context "kind-$CLUSTER_NAME" wait nodes --all --for=condition=Ready --timeout=120s - name: Export kind kubeconfig to mise path shell: bash diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index a9123fdaa8..819bfa4d32 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -463,6 +463,43 @@ jobs: agent-sandbox-version: ${{ matrix.agent_sandbox_version }} test-name: ${{ matrix.test_name }} + kubernetes-app-listener-e2e: + needs: [pr_metadata, build-binaries, build-images] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + strategy: + fail-fast: false + matrix: + include: + - topology: dedicated + delivery: push + takeover: graceful + name: dedicated push + - topology: dedicated + delivery: poll + takeover: graceful + name: dedicated poll + - topology: shared + delivery: push + takeover: graceful + name: shared graceful takeover + - topology: shared + delivery: push + takeover: abrupt + name: shared abrupt takeover + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + image-tag: ${{ github.sha }} + job-name: Kubernetes app listener (${{ matrix.name }}) + e2e-task: e2e:kubernetes:app-listener + enforce-network-policy: true + listener-topology: ${{ matrix.topology }} + listener-delivery: ${{ matrix.delivery }} + listener-takeover: ${{ matrix.takeover }} + kubernetes-workspace-managed-e2e: needs: [pr_metadata, build-binaries, build-images] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -552,6 +589,7 @@ jobs: - podman-external-driver-e2e - vm-external-driver-e2e - kubernetes-e2e + - kubernetes-app-listener-e2e - kubernetes-external-driver-e2e - kubernetes-workspace-managed-e2e - kubernetes-workspace-operator-e2e diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index cf6eb07b2d..867b8c1dd7 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -77,6 +77,23 @@ on: required: false type: string default: "gateway sandbox supervisor" + enforce-network-policy: + description: "Install a policy-enforcing CNI for listener isolation checks" + required: false + type: boolean + default: false + listener-topology: + required: false + type: string + default: "dedicated" + listener-delivery: + required: false + type: string + default: "push" + listener-takeover: + required: false + type: string + default: "graceful" permissions: actions: read @@ -129,6 +146,14 @@ jobs: registry-username: ${{ github.actor }} registry-password: ${{ secrets.GITHUB_TOKEN }} mise-version: ${{ inputs.mise-version }} + enforce-network-policy: ${{ inputs.enforce-network-policy }} + + - name: Preload listener workload and client fixture + if: inputs.e2e-task == 'e2e:kubernetes:app-listener' + run: | + docker pull --platform linux/amd64 python:3.12-slim + docker image save --platform linux/amd64 --output "$RUNNER_TEMP/listener-python.tar" python:3.12-slim + kind load image-archive "$RUNNER_TEMP/listener-python.tar" --name "$KIND_CLUSTER_NAME" - name: Run Kubernetes E2E env: @@ -142,4 +167,15 @@ jobs: IMAGE_TAG: ${{ inputs.image-tag }} OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell E2E_TASK: ${{ inputs.e2e-task }} + OPENSHELL_E2E_APP_TOPOLOGY: ${{ inputs.listener-topology }} + OPENSHELL_E2E_APP_DELIVERY: ${{ inputs.listener-delivery }} + OPENSHELL_E2E_APP_TAKEOVER: ${{ inputs.listener-takeover }} run: mise run --no-deps --skip-deps "$E2E_TASK" + + - name: Upload listener test evidence + if: always() && inputs.e2e-task == 'e2e:kubernetes:app-listener' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: listener-${{ inputs.listener-topology }}-${{ inputs.listener-delivery }}-${{ inputs.listener-takeover }} + path: results/app-listener/ + if-no-files-found: warn diff --git a/e2e/kubernetes/app-listener/README.md b/e2e/kubernetes/app-listener/README.md index 3e26d81db6..c3c5a65623 100644 --- a/e2e/kubernetes/app-listener/README.md +++ b/e2e/kubernetes/app-listener/README.md @@ -5,6 +5,101 @@ images. This fixture exercises the listener without discovery or the reference proxy. The opt-in Helm values are `deploy/helm/openshell/ci/values-app-listener.yaml`. +## Repeatable local checks + +The local and CI entry point creates an isolated Helm release, temporary TLS +Secret and client resources, runs the checks, and removes the release/namespace: + +```shell +KUBECONFIG="$PWD/kubeconfig" \ +IMAGE_TAG=dev OPENSHELL_REGISTRY=openshell \ +OPENSHELL_E2E_APP_TOPOLOGY=shared \ +OPENSHELL_E2E_APP_TAKEOVER=abrupt \ + mise run e2e:kubernetes:app-listener +``` + +Build and import matching `openshell/gateway:dev`, `openshell/supervisor:dev`, +and `openshell/sandbox:dev` images first, or set the registry/tag to images +already loaded into the cluster. Build the CLI or set `OPENSHELL_BIN` to it. +Set `OPENSHELL_E2E_KUBE_CONTEXT` to select a context explicitly. The cluster must +enforce NetworkPolicy; the local k3s default does. The task installs the pinned +Agent Sandbox controller only when its CRD is absent. +Set `OPENSHELL_E2E_KUBECONFIG` when testing a separate cluster whose kubeconfig +differs from the repository's mise environment. + +`OPENSHELL_E2E_APP_TOPOLOGY` accepts `dedicated` (default) or `shared`; +`OPENSHELL_E2E_APP_DELIVERY` accepts `push` (default) or `poll`; +`OPENSHELL_E2E_APP_TAKEOVER` accepts `graceful` (default) or `abrupt` for shared +supervisors. Dedicated runs include gateway-outage expiry. Every run also +disables the listener and verifies absent app sockets/TLS mounts/Services and +continued gateway application ingress. Logs, events, and resource summaries +are written under `results/app-listener/`, without TLS private keys or Secrets. + +GitHub Actions runs four matrix cases under the existing `test:e2e` label: +dedicated push/poll and shared graceful/abrupt takeover. These jobs use the +reusable Kubernetes workflow, with kindnet replaced by pinned Calico for policy +enforcement, and participate in `Core E2E result`. CI uploads the evidence even +on failure. They still require the normal current-head `/ok to test ` +approval mirror. + +### Checking an already deployed release + +Build the gateway, supervisor, sandbox, and CLI from the branch under test and +deploy them through the local Helm development workflow. Use an isolated release +namespace with gateway TLS, unauthenticated development API access, and the app +listener enabled. Provide the application TLS Secret before installing the +release. The workload and client fixture use `python:3.12-slim`. + +Run from the repository root, supplying the public application certificate/CA: + +```shell +KUBECONFIG="$PWD/kubeconfig" uv run --no-project python \ + e2e/kubernetes/app-listener/verify.py \ + --namespace listener-review-shared \ + --gateway-endpoint https://localhost:29203 \ + --forward-gateway service/listener-review \ + --app-ca /path/to/app-ca.crt \ + --network-policy \ + --takeover graceful +``` + +Use the actual namespace and gateway Service name of the test release. The +runner maintains its API port-forward across gateway restarts; application +traffic originates in client Pods and uses cluster networking. Application TLS +verifies the supplied CA and hostname. The development API connection uses +`--gateway-insecure` with an isolated temporary CLI configuration. + +The runner creates and cleans up its sandboxes, client Pods, public-CA ConfigMap, +and ingress policies. It checks named/unnamed HTTP routes, public Host/Origin and +cookies, credential stripping and explicit bearer passthrough, unknown routes, +Host/SNI mismatch, quiet WebSockets beyond 30 seconds, selective withdrawal, and +re-exposure without restarting the application. Both topologies use supervisor +Pod IPs and verify that no per-sandbox app Services are created. + +`--network-policy` requires an enforcing cluster network implementation, such as +the local k3s default. A default kind cluster needs a policy-enforcing CNI before +this assertion can establish isolation. + +For a two-member shared pool, `--takeover graceful` deletes the owning Pod and +checks that its sandbox keeps its Pod UID, runtime generation, process, and +in-memory state, while another owner's WebSocket survives. `--takeover abrupt` +first SIGKILLs the supervisor container through the local k3d/kind node's Docker +container, preventing a graceful leave; it requires access to that Docker engine. +These options deliberately interrupt test resources in the supplied namespace. + +For dedicated topology, omit `--takeover`. Add +`--expire-gateway statefulset/listener-review` (using the actual workload name) +to scale the isolated gateway to zero, verify route/stream expiry, restore its +original replica count, and check application survival. Repeat with gateway +`config_delivery_mode` set to `push` and `poll`. + +The lower-level runner leaves the Helm release and TLS Secret in place. Use +`--disabled --gateway-service ` instead of takeover/expiry flags to +check an already disabled release. Configuration rejection and the additional +deployment cleanup scenarios remain in the manual checklist below. + +## Manual deployment and additional scenarios + Before deployment, provide an operator-owned `openshell-app-tls` Secret in the supervisor namespace with `tls.crt` and `tls.key` for `*.services.example.com`. Keep private keys outside the checkout. Allow a labeled @@ -20,12 +115,12 @@ this stage. Use these direct-listener hostnames instead: - `default--.services.example.com` for the unnamed service. - `default----web.services.example.com` for the named service. -Inspect supervisor Pods. Dedicated supervisors optionally expose -`os-app-`; inspect that Service's EndpointSlices. Shared supervisors -have no app Service: select the Pod IP actually owning the sandbox and use its -configured app port. Do not select an arbitrary member when more than one is -present. Check ownership through supervisor admission/release logs in this -stage; automated owner selection belongs to the later discovery API. +Inspect supervisor Pods. Both dedicated and shared supervisors expose the +listener directly on their Pod IPs without app Services. Select the Pod IP +actually owning the sandbox and use its configured app port. Do not select an +arbitrary member when more than one is present. Check ownership through +supervisor admission/release logs in this stage; automated owner selection +belongs to the later discovery API. From a permitted client Pod, mount only the public app certificate/CA and run: @@ -35,9 +130,8 @@ curl --cacert /test/app-ca.crt \ https://default--demo--web.services.example.com/ ``` -For a dedicated supervisor, repeat using its headless Service DNS name instead -of the Pod IP. Verify both paths through ordinary cluster networking. A -port-forward alone does not verify Service selection or NetworkPolicy. +Verify this path through ordinary cluster networking. A port-forward alone does +not verify NetworkPolicy. Verify HTTP and WebSockets, public Host/Origin, application session cookies, unknown SNI, mismatched Host, undeclared services, bearer stripping/passthrough, @@ -53,17 +147,9 @@ the remaining freshness lease once session loss is detected. Run with listener configuration omitted, explicitly disabled despite complete TLS/port options, and explicitly enabled. Disabled deployments have no app port, -TLS mount, or app Service. Retain the dedicated driver `create_service` option -while disabling and recreating a sandbox to verify cleanup of its previously -owned endpoint. Confirm gateway ingress still works in each mode. - -Dedicated endpoint reconciliation runs more than once during creation. Verify -that it accepts its own non-controller Sandbox owner reference and has Service -`patch`/`delete` permissions when `create_service` is configured. Foreign-owned -Services must be left untouched, and cleanup uses UID/resource-version -preconditions. +TLS mount, or app Service. Confirm gateway ingress still works in each mode. -For HA pool validation after integration with #4864, use at least two supervisor +For HA pool validation, use at least two supervisor members with distinct sandbox owners. Move one owner through graceful downscale and abrupt loss. Existing streams may disconnect; after selecting the new owner, new HTTP/WebSocket connections must work without restarting the workload. diff --git a/e2e/kubernetes/app-listener/app.py b/e2e/kubernetes/app-listener/app.py new file mode 100644 index 0000000000..afa676d68b --- /dev/null +++ b/e2e/kubernetes/app-listener/app.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Loopback HTTP/WebSocket fixture; identity and counter survive supervisor moves.""" + +import base64 +import hashlib +import json +import os +import struct +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +BOOT = str(uuid.uuid4()) + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + counter = 0 + + def do_GET(self): + Handler.counter += 1 + if self.headers.get("Upgrade", "").lower() != "websocket": + body = json.dumps( + { + "headers": dict(self.headers), + "boot": BOOT, + "pid": os.getpid(), + "counter": Handler.counter, + } + ).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + # SHA-1 is mandated by the WebSocket handshake, not used for authentication. + accept = base64.b64encode( + hashlib.sha1( + ( + self.headers["Sec-WebSocket-Key"] + + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + ).encode(), + usedforsecurity=False, + ).digest() + ).decode() + self.send_response(101) + self.send_header("Upgrade", "websocket") + self.send_header("Connection", "Upgrade") + self.send_header("Sec-WebSocket-Accept", accept) + self.end_headers() + while True: + head = self.rfile.read(2) + if len(head) != 2 or head[0] & 15 == 8: + return + length = head[1] & 127 + if length == 126: + length = struct.unpack("!H", self.rfile.read(2))[0] + if length > 125 or not head[1] & 128: + return # Fixture deliberately accepts only small masked frames. + mask = self.rfile.read(4) + data = self.rfile.read(length) + if len(mask) != 4 or len(data) != length: + return + data = bytes(value ^ mask[i % 4] for i, value in enumerate(data)) + self.wfile.write(bytes([0x81, len(data)]) + data) + self.wfile.flush() + + +ThreadingHTTPServer(("127.0.0.1", 8080), Handler).serve_forever() diff --git a/e2e/kubernetes/app-listener/client.py b/e2e/kubernetes/app-listener/client.py new file mode 100644 index 0000000000..e218835058 --- /dev/null +++ b/e2e/kubernetes/app-listener/client.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Executed in a client Pod so probes use actual cluster networking.""" + +import errno +import http.client +import json +import socket +import ssl +import sys +import time + +CONTEXT = ssl.create_default_context(cafile="/test/ca.crt") + + +def connect(ip, port, hostname, context=CONTEXT): + raw = socket.create_connection((ip, int(port)), timeout=4) + try: + return context.wrap_socket(raw, server_hostname=hostname) + except BaseException: + raw.close() + raise + + +def request( + ip, + port, + hostname, + authority=None, + context=CONTEXT, + authorization="Bearer application-test-token", +): + with connect(ip, port, hostname, context) as stream: + headers = { + "Host": authority or hostname, + "Origin": "https://" + hostname, + "Cookie": "app_session=fixture", + "X-Openshell-Principal": "must-strip", + "Connection": "close", + } + if authorization is not None: + headers["Authorization"] = authorization + stream.sendall( + ( + "GET / HTTP/1.1\r\n" + + "".join(f"{k}: {v}\r\n" for k, v in headers.items()) + + "\r\n" + ).encode() + ) + response = http.client.HTTPResponse(stream) + response.begin() + body = response.read() + return response.status, json.loads( + body + ) if response.status == 200 else body.decode() + + +def websocket(ip, port, hostname): + stream = connect(ip, port, hostname) + stream.sendall( + ( + f"GET /echo HTTP/1.1\r\nHost: {hostname}\r\nUpgrade: websocket\r\n" + "Connection: Upgrade\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + "Sec-WebSocket-Version: 13\r\n\r\n" + ).encode() + ) + response = b"" + while b"\r\n\r\n" not in response: + chunk = stream.recv(1) + assert chunk, "closed before WebSocket handshake" + response += chunk + assert len(response) < 8192 + assert b" 101 " in response, response + return stream + + +def exact(stream, count): + data = b"" + while len(data) < count: + chunk = stream.recv(count - len(data)) + assert chunk, "WebSocket closed unexpectedly" + data += chunk + return data + + +def echo(stream): + stream.sendall( + b"\x81\x84abcd" + bytes(c ^ m for c, m in zip(b"echo", b"abcd", strict=False)) + ) + assert exact(stream, 6) == b"\x81\x04echo" + + +def closed(stream): + stream.settimeout(40) + try: + data = stream.recv(2) + assert not data or data[0] & 15 == 8, data + except (ssl.SSLError, ConnectionError): + pass + + +def main(): + mode, ip, port, hostname = sys.argv[1:5] + if mode == "http": + print(json.dumps(request(ip, port, hostname))) + elif mode == "reject": + try: + status, _ = request(ip, port, hostname) + assert status != 200, "unexpected successful route" + except (ssl.SSLError, ConnectionError, http.client.HTTPException): + pass + print("PASS undeclared SNI rejected") + elif mode == "blocked": + try: + with socket.create_connection((ip, int(port)), timeout=4): + raise AssertionError("unpermitted client reached application port") + except (TimeoutError, ConnectionRefusedError): + pass + print("PASS unpermitted client blocked by network policy") + elif mode == "disabled": + with socket.socket() as stream: + stream.settimeout(4) + result = stream.connect_ex((ip, int(port))) + assert result == errno.ECONNREFUSED, (ip, result) + print("PASS disabled listener refuses TCP from the node network") + elif mode == "gateway": + status, body = request( + ip, + port, + hostname, + context=ssl._create_unverified_context(), + authorization=None, + ) + assert status == 200 and body["boot"], (status, body) + print("PASS gateway-mediated application ingress with listener disabled") + elif mode == "headers": + status, body = request(ip, port, hostname) + assert status == 200, (status, body) + headers = {k.lower(): v for k, v in body["headers"].items()} + assert headers["host"] == hostname + assert headers["origin"] == "https://" + hostname + assert headers["cookie"] == "app_session=fixture" + assert "authorization" not in headers + assert "x-openshell-principal" not in headers + status, _ = request(ip, port, hostname, "different.services.example.com") + assert status in (400, 421), status + print( + "PASS public authority/cookies preserved, credentials stripped, mismatched Host rejected" + ) + elif mode == "bearer": + status, body = request(ip, port, hostname) + assert status == 200 + headers = {k.lower(): v for k, v in body["headers"].items()} + assert headers["authorization"] == "Bearer application-test-token" + print("PASS explicitly configured bearer passthrough") + elif mode in ("withdraw", "takeover", "expire"): + with ( + websocket(ip, port, hostname) as affected, + websocket(*sys.argv[5:8]) as unaffected, + ): + echo(affected) + echo(unaffected) + time.sleep(35) + echo(affected) + echo(unaffected) + print( + "READY: quiet WebSockets survived beyond freshness window", flush=True + ) + closed(affected) + if mode != "expire": + echo(unaffected) + else: + closed(unaffected) + print( + "PASS affected streams closed" + + ("; unrelated stream survived" if mode != "expire" else "") + ) + else: + raise ValueError(mode) + + +if __name__ == "__main__": + main() diff --git a/e2e/kubernetes/app-listener/run.py b/e2e/kubernetes/app-listener/run.py new file mode 100644 index 0000000000..0e0b954349 --- /dev/null +++ b/e2e/kubernetes/app-listener/run.py @@ -0,0 +1,367 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Deploy and verify an isolated listener release on the selected local/CI cluster.""" + +import argparse +import json +import os +import socket +import subprocess +import sys +import tempfile +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +HERE = Path(__file__).resolve().parent + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--topology", + choices=("shared", "dedicated"), + default=os.getenv("OPENSHELL_E2E_APP_TOPOLOGY", "dedicated"), + ) + parser.add_argument( + "--delivery", + choices=("push", "poll"), + default=os.getenv("OPENSHELL_E2E_APP_DELIVERY", "push"), + ) + parser.add_argument( + "--takeover", + choices=("graceful", "abrupt"), + default=os.getenv("OPENSHELL_E2E_APP_TAKEOVER", "graceful"), + ) + parser.add_argument("--image-tag", default=os.getenv("IMAGE_TAG", "dev")) + parser.add_argument( + "--registry", default=os.getenv("OPENSHELL_REGISTRY", "openshell") + ) + args = parser.parse_args() + namespace = "listener-e2e-" + uuid.uuid4().hex[:8] + context = os.getenv("OPENSHELL_E2E_KUBE_CONTEXT") + kubeconfig = os.getenv("OPENSHELL_E2E_KUBECONFIG") + config_args = ["--kubeconfig", kubeconfig] if kubeconfig else [] + kube_args = [*config_args, *(["--context", context] if context else [])] + helm_args = [*config_args, *(["--kube-context", context] if context else [])] + k = ["kubectl", *kube_args, "-n", namespace] + helm = ["helm", *helm_args] + cli = Path(os.getenv("OPENSHELL_BIN", ROOT / "target/debug/openshell")).resolve() + evidence = ( + ROOT + / "results/app-listener" + / f"{args.topology}-{args.delivery}-{args.takeover}" + ) + evidence.mkdir(parents=True, exist_ok=True) + + def run(command, *, input=None, check=True, stdout=None, stderr=None): + result = subprocess.run( + command, + input=input, + text=True, + check=False, + stdout=stdout, + stderr=stderr, + timeout=600, + ) + if check and result.returncode: + raise RuntimeError(f"{command[:6]} exited {result.returncode}") + return result + + def capture(command, destination): + with (evidence / destination).open("w") as output: + run(command, check=False, stdout=output) + + def wait_for_shared_hosts(): + if args.topology != "shared": + return + # TCP readiness precedes token projection and authenticated attachment. + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + listing = subprocess.run( + [ + *k, + "get", + "pods", + "-l", + "app.kubernetes.io/component=supervisor", + "-o", + "json", + ], + capture_output=True, + text=True, + timeout=30, + ) + members = ( + json.loads(listing.stdout)["items"] if listing.returncode == 0 else [] + ) + attached = 0 + for member in members: + logs = subprocess.run( + [*k, "logs", member["metadata"]["name"], "-c", "supervisor"], + capture_output=True, + text=True, + timeout=30, + ) + attached += "gateway accepted the supervisor host" in logs.stdout + if len(members) == 2 and attached == 2: + return + time.sleep(2) + raise AssertionError( + "both supervisor hosts must authenticate before creating sandboxes" + ) + + if subprocess.run( + ["kubectl", *kube_args, "get", "crd/sandboxes.agents.x-k8s.io"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + ).returncode: + run(["bash", str(ROOT / "e2e/support/install-agent-sandbox.sh"), *kube_args]) + assert cli.is_file(), f"Build the CLI or set OPENSHELL_BIN: {cli}" + run(["kubectl", *kube_args, "create", "namespace", namespace]) + try: + with tempfile.TemporaryDirectory(prefix="openshell-listener-tls-") as temporary: + material = Path(temporary) + run( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + str(material / "app.key"), + "-out", + str(material / "app.crt"), + "-days", + "2", + "-subj", + "/CN=*.services.example.com", + "-addext", + "subjectAltName=DNS:*.services.example.com", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + (material / "app.key").chmod(0o600) + run( + [ + *k, + "create", + "secret", + "tls", + "openshell-app-tls", + "--cert", + str(material / "app.crt"), + "--key", + str(material / "app.key"), + ] + ) + values = { + "fullnameOverride": "listener-e2e", + "global": { + "image": { + "registry": "", + "tag": args.image_tag, + "pullPolicy": "Never", + } + }, + "gateway": {"image": {"repository": args.registry + "/gateway"}}, + "supervisor": { + "image": {"repository": args.registry + "/supervisor"}, + "replicas": 2, + "recoveryTimeoutSeconds": 60, + }, + "sandboxRuntime": {"image": {"repository": args.registry + "/sandbox"}}, + "sandbox": { + "image": { + "repository": "python", + "tag": "3.12-slim", + "pullPolicy": "IfNotPresent", + } + }, + "server": { + "auth": {"allowUnauthenticatedUsers": True}, + "tls": {"enableMtls": False}, + }, + "gatewayConfig": { + "openshell.gateway": {"config_delivery_mode": args.delivery}, + "openshell.drivers.kubernetes": {"supervisor_topology": "gateway"} + if args.topology == "shared" + else {}, + "openshell.supervisor.app_listener": { + "enabled": True, + "port": 9443, + "domain": "services.example.com", + }, + "openshell.drivers.kubernetes.sandbox_runtime.app_listener": { + "tls_secret_name": "openshell-app-tls", + }, + }, + } + value_file = material / "values.json" + value_file.write_text(json.dumps(values)) + run( + [ + *helm, + "upgrade", + "--install", + "listener-e2e", + str(ROOT / "deploy/helm/openshell"), + "-n", + namespace, + "-f", + str(value_file), + "--wait", + "--timeout", + "5m", + ] + ) + wait_for_shared_hosts() + # The port is used only for API forwarding; app clients run inside Pods. + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + env = dict(os.environ) + if context or kubeconfig: + env["KUBECONFIG"] = str(material / "kubeconfig") + with Path(env["KUBECONFIG"]).open("w") as output: + run( + [ + "kubectl", + *kube_args, + "config", + "view", + "--minify", + "--raw", + "--flatten", + ], + stdout=output, + ) + command = [ + sys.executable, + str(HERE / "verify.py"), + "--namespace", + namespace, + "--gateway-endpoint", + f"https://localhost:{port}", + "--forward-gateway", + "service/listener-e2e", + "--cli", + str(cli), + "--app-ca", + str(material / "app.crt"), + "--network-policy", + ] + if args.topology == "shared": + command += ["--takeover", args.takeover] + else: + command += ["--expire-gateway", "statefulset/listener-e2e"] + with (evidence / "enabled.log").open("w") as output: + result = subprocess.run( + command, + env=env, + text=True, + stdout=output, + stderr=subprocess.STDOUT, + timeout=900, + ) + print((evidence / "enabled.log").read_text(), flush=True) + assert result.returncode == 0, "enabled-listener checks failed" + disabled = material / "disabled.json" + disabled.write_text( + json.dumps( + { + "gatewayConfig": { + "openshell.supervisor.app_listener": {"enabled": False} + } + } + ) + ) + run( + [ + *helm, + "upgrade", + "listener-e2e", + str(ROOT / "deploy/helm/openshell"), + "-n", + namespace, + "--reuse-values", + "-f", + str(disabled), + "--wait", + "--timeout", + "3m", + ] + ) + wait_for_shared_hosts() + # A separate pass verifies actual disabled sockets and legacy ingress. + disabled_command = [ + *command[: command.index("--network-policy")], + "--disabled", + "--gateway-service", + "listener-e2e", + ] + with (evidence / "disabled.log").open("w") as output: + result = subprocess.run( + disabled_command, + env=env, + text=True, + stdout=output, + stderr=subprocess.STDOUT, + timeout=300, + ) + print((evidence / "disabled.log").read_text(), flush=True) + assert result.returncode == 0, "disabled-listener checks failed" + finally: + capture( + [*k, "get", "pods,services,endpointslices,networkpolicies", "-o", "wide"], + "resources.log", + ) + capture([*k, "get", "events", "--sort-by=.lastTimestamp"], "events.log") + pods = subprocess.run( + [*k, "get", "pods", "-o", "json"], + capture_output=True, + text=True, + timeout=30, + ) + if pods.returncode == 0: + for pod in json.loads(pods.stdout)["items"]: + for container in pod["spec"]["containers"]: + if container["name"] in ("supervisor", "openshell-gateway"): + capture( + [ + *k, + "logs", + pod["metadata"]["name"], + "-c", + container["name"], + "--tail=1000", + ], + pod["metadata"]["name"] + ".log", + ) + run( + [ + *helm, + "uninstall", + "listener-e2e", + "-n", + namespace, + "--wait", + "--timeout", + "2m", + ], + check=False, + ) + run( + ["kubectl", *kube_args, "delete", "namespace", namespace, "--wait=false"], + check=False, + ) + + +if __name__ == "__main__": + main() diff --git a/e2e/kubernetes/app-listener/verify.py b/e2e/kubernetes/app-listener/verify.py new file mode 100644 index 0000000000..e6e80d59c4 --- /dev/null +++ b/e2e/kubernetes/app-listener/verify.py @@ -0,0 +1,683 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Verify an isolated Helm listener release, without discovery or a proxy. + +Run with uv run --no-project python; see README.md for deployment prerequisites. +""" + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import threading +import time +import uuid +from contextlib import ExitStack +from pathlib import Path +from urllib.parse import urlsplit + +HERE = Path(__file__).resolve().parent + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--namespace", required=True) + parser.add_argument("--gateway-endpoint", required=True) + parser.add_argument( + "--forward-gateway", + help="Maintain an API port-forward to this Service across gateway restarts (e.g. service/listener-review)", + ) + parser.add_argument("--app-ca", type=Path, required=True) + parser.add_argument("--cli", default="target/debug/openshell") + parser.add_argument("--domain", default="services.example.com") + parser.add_argument("--port", default="9443") + parser.add_argument( + "--disabled", + action="store_true", + help="Check a disabled listener and continued gateway application ingress", + ) + parser.add_argument( + "--gateway-service", help="Gateway Service used by the disabled-listener check" + ) + parser.add_argument( + "--takeover", + nargs="?", + const="graceful", + choices=("graceful", "abrupt"), + help="Stop a shared owner Pod; abrupt requires Docker access to the k3d/kind node", + ) + parser.add_argument( + "--expire-gateway", + help="Scale this isolated gateway workload to zero, verify expiry, then restore it (e.g. statefulset/listener-review)", + ) + parser.add_argument( + "--network-policy", + action="store_true", + help="Require denied-client enforcement", + ) + args = parser.parse_args() + if args.disabled and not args.gateway_service: + parser.error("--disabled requires --gateway-service") + prefix = "listener-" + uuid.uuid4().hex[:6] + k = ["kubectl", "-n", args.namespace] + cli = [ + str(Path(args.cli).resolve()), + "--gateway-endpoint", + args.gateway_endpoint, + "--gateway-insecure", + ] + created = [] + + def run(command, *, check=True, timeout=180, input=None): + result = subprocess.run( + command, text=True, capture_output=True, timeout=timeout, input=input + ) + if check and result.returncode: + raise RuntimeError(f"{command[:7]}: {result.stdout}\n{result.stderr}") + return result + + def apply(obj): + run([*k, "apply", "-f", "-"], input=json.dumps(obj)) + + def probe(mode, ip, host, *extra, client="allowed"): + return run( + [ + *k, + "exec", + prefix + "-" + client, + "--", + "python", + "/test/client.py", + mode, + ip, + args.port, + host, + *extra, + ], + timeout=60, + ) + + def hostname(name, service="web"): + return ( + f"default--{name}" + (f"--{service}" if service else "") + "." + args.domain + ) + + def pods(): + items = json.loads(run([*k, "get", "pods", "-o", "json"]).stdout)["items"] + return [ + pod + for pod in items + if pod["status"].get("podIP") + and not pod["metadata"].get("deletionTimestamp") + and any(c["name"] == "supervisor" for c in pod["spec"]["containers"]) + ] + + def owner(name, timeout=120): + end = time.monotonic() + timeout + while time.monotonic() < end: + matches = [] + for pod in pods(): + ip = pod["status"]["podIP"] + result = run( + [ + *k, + "exec", + prefix + "-allowed", + "--", + "python", + "/test/client.py", + "http", + ip, + args.port, + hostname(name), + ], + check=False, + timeout=20, + ) + if result.returncode == 0: + status, body = json.loads(result.stdout) + if status == 200: + matches.append((pod, body)) + if len(matches) == 1: + return matches[0] + assert len(matches) < 2, "multiple supervisors serve one sandbox" + time.sleep(1) + raise AssertionError(f"no unique serving owner for {name}") + + def create(index, wait=True): + name = f"{prefix}-{index}" + created.append(name) + run( + [ + *cli, + "sandbox", + "create", + "--name", + name, + "--from", + "python:3.12-slim", + "--detach", + "--expose", + "8080", + "--", + "python", + "-c", + (HERE / "app.py").read_text(), + ] + ) + run([*cli, "service", "expose", name, "8080", "web"]) + return (name, *owner(name)) if wait else name + + def check_disabled(): + name = create(0, wait=False) + deadline = time.monotonic() + 120 + while True: + members = pods() + if members and all(p["status"].get("phase") == "Running" for p in members): + break + assert time.monotonic() < deadline, "disabled supervisor did not start" + time.sleep(1) + for member in members: + container = next( + c for c in member["spec"]["containers"] if c["name"] == "supervisor" + ) + assert not any(p.get("name") == "app" for p in container.get("ports", [])) + assert not any( + v["name"] == "app-tls" for v in container.get("volumeMounts", []) + ) + assert not any( + e["name"] == "OPENSHELL_APP_LISTENER_ENABLED" + and e.get("value") == "true" + for e in container.get("env", []) + ) + print( + probe( + "disabled", member["status"]["podIP"], hostname(name), client="host" + ).stdout, + end="", + flush=True, + ) + services = json.loads( + run( + [ + *k, + "get", + "services", + "-l", + "openshell.ai/component=app-service", + "-o", + "json", + ] + ).stdout + ) + assert not services["items"], "disabled deployment created app Services" + gateway = json.loads( + run([*k, "get", "service", args.gateway_service, "-o", "json"]).stdout + ) + deadline = time.monotonic() + 120 + while True: + result = run( + [ + *k, + "exec", + prefix + "-host", + "--", + "python", + "/test/client.py", + "gateway", + gateway["spec"]["clusterIP"], + "8080", + f"default--{name}--web.openshell.localhost", + ], + check=False, + ) + if result.returncode == 0: + print(result.stdout, end="", flush=True) + break + assert time.monotonic() < deadline, result.stderr + time.sleep(1) + print( + "PASS disabled listener omits TLS material/port and preserves gateway access", + flush=True, + ) + + def identity(name): + obj = json.loads( + run([*k, "get", "sandbox", "default--" + name, "-o", "json"]).stdout + ) + pod = json.loads( + run([*k, "get", "pod", "default--" + name, "-o", "json"]).stdout + ) + return ( + obj["metadata"]["uid"], + pod["metadata"]["uid"], + obj["metadata"]["annotations"]["openshell.ai/sandbox-runtime-generation"], + ) + + def stop_owner(pod): + name = pod["metadata"]["name"] + if args.takeover == "abrupt": + container = next( + c["containerID"].split("://", 1)[1] + for c in pod["status"]["containerStatuses"] + if c["name"] == "supervisor" + ) + for ctr in (["ctr"], ["k3s", "ctr"]): + result = run( + [ + "docker", + "exec", + pod["spec"]["nodeName"], + *ctr, + "-n", + "k8s.io", + "tasks", + "kill", + "--signal", + "SIGKILL", + container, + ], + check=False, + ) + if result.returncode == 0: + break + assert result.returncode == 0, result.stderr + run( + [ + *k, + "delete", + "pod", + name, + "--grace-period=0", + "--force", + "--wait=false", + ] + ) + else: + run([*k, "delete", "pod", name, "--wait=false"]) + + def streams(mode, ip, first, second_ip, second, action): + process = subprocess.Popen( + [ + *k, + "exec", + prefix + "-allowed", + "--", + "python", + "-u", + "/test/client.py", + mode, + ip, + args.port, + first, + second_ip, + args.port, + second, + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + try: + # Bound output waits too: a stalled kubectl exec must not hang the suite. + import selectors + + ready = False + end = time.monotonic() + 110 + assert process.stdout is not None + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + while process.poll() is None and time.monotonic() < end: + if not selector.select(timeout=1): + continue + line = process.stdout.readline() + print(line, end="", flush=True) + if line.startswith("READY:"): + ready = True + action() + tail, _ = process.communicate(timeout=5) + print(tail, end="", flush=True) + assert ready and process.returncode == 0, ( + f"{mode}: exit={process.returncode}" + ) + finally: + if process.poll() is None: + process.kill() + process.wait() + + with ( + tempfile.TemporaryDirectory(prefix="openshell-listener-client-") as config, + ExitStack() as stack, + ): + # Never change or depend on the developer's selected gateway/login. + previous = os.environ.get("XDG_CONFIG_HOME") + os.environ["XDG_CONFIG_HOME"] = config + if args.forward_gateway: + endpoint = urlsplit(args.gateway_endpoint) + assert ( + endpoint.hostname in ("localhost", "127.0.0.1", "::1") and endpoint.port + ) + stopped = threading.Event() + + def forward(): + with (Path(config) / "forward.log").open("w") as log: + while not stopped.is_set(): + with subprocess.Popen( + [ + *k, + "port-forward", + args.forward_gateway, + f"{endpoint.port}:8080", + ], + stdout=log, + stderr=subprocess.STDOUT, + ) as child: + while child.poll() is None and not stopped.wait(0.2): + pass + if child.poll() is None: + child.terminate() + try: + child.wait(timeout=5) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + stopped.wait(1) + + forwarding = threading.Thread(target=forward) + forwarding.start() + + def stop_forward(): + stopped.set() + forwarding.join(timeout=10) + assert not forwarding.is_alive(), "API port-forward did not stop" + + stack.callback(stop_forward) + try: + deadline = time.monotonic() + 30 + while True: + ready = run([*cli, "sandbox", "list"], check=False, timeout=15) + if ready.returncode == 0: + break + assert time.monotonic() < deadline, ready.stderr + time.sleep(1) + apply( + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": prefix, "labels": {"listener-e2e": prefix}}, + "data": { + "ca.crt": args.app_ca.read_text(), + "client.py": (HERE / "client.py").read_text(), + }, + } + ) + for role in ("host",) if args.disabled else ("allowed", "denied"): + apply( + { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": prefix + "-" + role, + "labels": {"listener-e2e": prefix, "listener-client": role}, + }, + "spec": { + "hostNetwork": role == "host", + "dnsPolicy": "ClusterFirstWithHostNet" + if role == "host" + else "ClusterFirst", + "containers": [ + { + "name": "client", + "image": "python:3.12-slim", + "command": ["sleep", "infinity"], + "volumeMounts": [ + { + "name": "test", + "mountPath": "/test", + "readOnly": True, + } + ], + } + ], + "volumes": [ + {"name": "test", "configMap": {"name": prefix}} + ], + }, + } + ) + for index, labels in enumerate( + ( + {"app.kubernetes.io/component": "supervisor"}, + {"openshell.ai/component": "supervisor"}, + ) + ): + apply( + { + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": prefix + str(index), + "labels": {"listener-e2e": prefix}, + }, + "spec": { + "podSelector": {"matchLabels": labels}, + "policyTypes": ["Ingress"], + "ingress": [ + { + "from": [ + { + "podSelector": { + "matchLabels": { + "listener-e2e": prefix, + "listener-client": "allowed", + } + } + } + ], + "ports": [ + {"protocol": "TCP", "port": int(args.port)} + ], + } + ], + }, + } + ) + run( + [ + *k, + "wait", + "pod", + "-l", + "listener-e2e=" + prefix, + "--for=condition=Ready", + "--timeout=120s", + ] + ) + if args.disabled: + check_disabled() + return + name, pod, body = create(0) + ip = pod["status"]["podIP"] + host = hostname(name) + print("PASS unique serving supervisor and named HTTP route", flush=True) + services = json.loads(run([*k, "get", "services", "-o", "json"]).stdout) + assert not any( + service["metadata"]["name"].startswith("os-app-") + or service["metadata"].get("labels", {}).get("openshell.ai/component") + == "app-service" + for service in services["items"] + ), "listener deployment created app Services" + print("PASS Pod-IP ingress without per-sandbox app Services", flush=True) + assert json.loads(probe("http", ip, hostname(name, "")).stdout)[0] == 200 + print("PASS unnamed HTTP route", flush=True) + print(probe("headers", ip, host).stdout, end="", flush=True) + print( + probe("reject", ip, hostname(name, "missing")).stdout, + end="", + flush=True, + ) + if args.network_policy: + print( + probe("blocked", ip, host, client="denied").stdout, + end="", + flush=True, + ) + run( + [ + *cli, + "service", + "expose", + name, + "8080", + "token", + "--authorization-mode", + "bearer-passthrough", + ] + ) + deadline = time.monotonic() + 30 + while True: + result = run( + [ + *k, + "exec", + prefix + "-allowed", + "--", + "python", + "/test/client.py", + "bearer", + ip, + args.port, + hostname(name, "token"), + ], + check=False, + ) + if result.returncode == 0: + print(result.stdout, end="", flush=True) + break + assert time.monotonic() < deadline, result.stderr + time.sleep(1) + streams( + "withdraw", + ip, + host, + ip, + hostname(name, ""), + lambda: run([*cli, "service", "delete", name, "web"]), + ) + print(probe("reject", ip, host).stdout, end="", flush=True) + run([*cli, "service", "expose", name, "8080", "web"]) + _, current = owner(name) + assert current["boot"] == body["boot"] and current["pid"] == body["pid"] + print("PASS re-exposure preserves running application", flush=True) + if args.takeover: + peer = None + for index in range(1, 33): + candidate = create(index) + if candidate[1]["metadata"]["name"] != pod["metadata"]["name"]: + peer = candidate + break + assert peer is not None, ( + "could not place a sandbox on another supervisor" + ) + other, other_pod, other_body = peer + before = identity(name) + streams( + "takeover", + ip, + host, + other_pod["status"]["podIP"], + hostname(other), + lambda: stop_owner(pod), + ) + new_pod, after = owner(name, timeout=180) + assert new_pod["metadata"]["uid"] != pod["metadata"]["uid"] + assert identity(name) == before + assert (after["boot"], after["pid"]) == (body["boot"], body["pid"]) + assert after["counter"] > body["counter"] + peer_pod, peer_after = owner(other) + assert peer_pod["metadata"]["uid"] == other_pod["metadata"]["uid"] + assert peer_after["boot"] == other_body["boot"] + print( + f"PASS {args.takeover} owner takeover preserves workload Pod, generation, process and memory; peer unaffected", + flush=True, + ) + if args.expire_gateway: + obj = json.loads( + run([*k, "get", args.expire_gateway, "-o", "json"]).stdout + ) + replicas = obj["spec"]["replicas"] + assert replicas > 0 + expiry_pod, _ = owner(name) + expiry_ip = expiry_pod["status"]["podIP"] + try: + streams( + "expire", + expiry_ip, + host, + expiry_ip, + hostname(name, ""), + lambda: run([*k, "scale", args.expire_gateway, "--replicas=0"]), + ) + print(probe("reject", expiry_ip, host).stdout, end="", flush=True) + finally: + run([*k, "scale", args.expire_gateway, f"--replicas={replicas}"]) + run( + [*k, "rollout", "status", args.expire_gateway, "--timeout=180s"] + ) + _, resumed = owner(name) + assert (resumed["boot"], resumed["pid"]) == (body["boot"], body["pid"]) + print( + "PASS gateway outage expires streams and routes; recovery preserves application", + flush=True, + ) + print("PASS listener Kubernetes e2e", flush=True) + finally: + primary_failure = sys.exc_info()[0] is not None + cleanup_failures = [] + for name in reversed(created): + deadline = time.monotonic() + 60 + while True: + try: + deleted = run( + [*cli, "sandbox", "delete", name], check=False, timeout=10 + ) + if ( + deleted.returncode == 0 + or "not found" in (deleted.stdout + deleted.stderr).lower() + ): + break + detail = deleted.stderr + except subprocess.TimeoutExpired: + detail = "API request timed out during reconnection" + if time.monotonic() >= deadline: + cleanup_failures.append(name) + print( + f"Failed to clean sandbox {name}: {detail}", + file=sys.stderr, + ) + break + time.sleep(1) + run( + [ + *k, + "delete", + "pod,configmap,networkpolicy", + "-l", + "listener-e2e=" + prefix, + "--wait=false", + ], + check=False, + ) + if previous is None: + os.environ.pop("XDG_CONFIG_HOME", None) + else: + os.environ["XDG_CONFIG_HOME"] = previous + if cleanup_failures and not primary_failure: + raise AssertionError(f"sandbox cleanup failed: {cleanup_failures}") + + +if __name__ == "__main__": + main() diff --git a/mise.toml b/mise.toml index 9504f82781..10c964e956 100644 --- a/mise.toml +++ b/mise.toml @@ -74,7 +74,7 @@ DOCKER_BUILDKIT = "1" [vars] # Python paths to include in formatting/linting -python_paths = "python/ tasks/scripts/*.py e2e/mcp-conformance/*.py deploy/sbom/*.py" +python_paths = "python/ tasks/scripts/*.py e2e/mcp-conformance/*.py e2e/kubernetes/app-listener/*.py deploy/sbom/*.py" [task_config] includes = ["tasks/*.toml"] diff --git a/tasks/test.toml b/tasks/test.toml index 6bafa95e9e..ea4abc8bd9 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -286,6 +286,10 @@ description = "Run Kubernetes e2e with managed workspace mode (auto-created per- env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_CREDENTIAL_DRIVER = "kubernetes-secrets", OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } run = "e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:app-listener"] +description = "Deploy an isolated Helm release and verify application ingress, isolation, freshness, and supervisor ownership" +run = "uv run --no-project python e2e/kubernetes/app-listener/run.py" + ["e2e:kubernetes:shared-supervisor"] description = "Run Kubernetes e2e with a one-member shared supervisor pool for the gateway's sandboxes (gateway supervisor topology)" env = { OPENSHELL_E2E_KUBE_TEST = "kubernetes_shared_supervisor", OPENSHELL_E2E_KUBE_TLS = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-shared-supervisor" } From 8ad31623f6019ad64883995658d05bae2e93fdb7 Mon Sep 17 00:00:00 2001 From: divesh Date: Fri, 30 Oct 2026 00:04:52 -0700 Subject: [PATCH 3/5] fix(supervisor): enforce app route limits and shutdown fencing Reject excess service declarations before persistence, fence registration against shutdown, and require the expected application Service count in listener E2E checks. Add focused regression coverage. Signed-off-by: divesh Signed-off-by: Taylor Mutch --- crates/openshell-server/src/grpc/service.rs | 197 ++++++++++++++++-- .../openshell-supervisor/src/app_listener.rs | 61 +++++- e2e/kubernetes/app-listener/README.md | 1 + e2e/kubernetes/app-listener/run.py | 2 + e2e/kubernetes/app-listener/verify.py | 17 +- e2e/kubernetes/app-listener/verify_test.py | 28 +++ tasks/test.toml | 5 +- 7 files changed, 284 insertions(+), 27 deletions(-) create mode 100644 e2e/kubernetes/app-listener/verify_test.py diff --git a/crates/openshell-server/src/grpc/service.rs b/crates/openshell-server/src/grpc/service.rs index a47791e8d8..bea999e5af 100644 --- a/crates/openshell-server/src/grpc/service.rs +++ b/crates/openshell-server/src/grpc/service.rs @@ -19,11 +19,13 @@ use crate::ServerState; use crate::auth::workspace_authz::{ AuthorizedWorkspaceScope, MinWorkspaceRole, authorize_list_workspace_selector, }; +use crate::compute::MutationScope; use crate::pagination::Pagination; use crate::persistence::{ObjectListQuery, ObjectType, WriteCondition}; use crate::service_routing; const MAX_SERVICE_NAME_LEN: usize = super::MAX_ROUTABLE_NAME_LEN; +const MAX_APP_SERVICE_ROUTES: u32 = 1024; #[cfg(test)] #[derive(Default)] @@ -92,6 +94,21 @@ pub(super) async fn expose_service_endpoint( authorization_mode: ServiceAuthorizationMode, ) -> Result, Status> { let sandbox_name = sandbox.object_name(); + // Count and persist under the same fleet-wide sandbox guard, including + // create-time exposure, so concurrent gateways cannot overfill the allowlist. + let _mutation_guard = if state.config.supervisor_app_listener.enabled { + Some( + state + .compute + .mutation_guard(MutationScope::sandbox(workspace, sandbox.object_id())) + .await + .map_err(|error| { + super::persistence_error_to_status(error, "acquire sandbox mutation lock") + })?, + ) + } else { + None + }; let now = crate::persistence::current_time_ms(); let key = service_routing::endpoint_key(sandbox_name, service); @@ -103,6 +120,15 @@ pub(super) async fn expose_service_endpoint( .await .map_err(|e| Status::internal(format!("fetch endpoint failed: {e}")))?; + if state.config.supervisor_app_listener.enabled + && existing.is_none() + && app_service_endpoints(state, sandbox).await?.len() >= MAX_APP_SERVICE_ROUTES as usize + { + return Err(Status::resource_exhausted( + "too many application service declarations", + )); + } + let (id, created_at_ms, condition, created) = if let Some(existing) = existing { // Update path: preserve id and created_at, use CAS to prevent conflicts let resource_version = existing @@ -383,26 +409,8 @@ pub(super) async fn configured_app_routes( if !state.config.supervisor_app_listener.enabled { return Ok(Vec::new()); } - let selector = format!("sandbox={}", sandbox.object_name()); - let page = state - .store - .list_message_page::( - ObjectListQuery::WorkspaceSelector { - workspace: sandbox.object_workspace(), - label_selector: &selector, - }, - None, - 1024, - ) - .await - .map_err(|_| Status::unavailable("service declarations unavailable"))?; - if page.next_cursor.is_some() { - return Err(Status::resource_exhausted( - "too many application service declarations", - )); - } - let mut service_routes: Vec<_> = page - .messages + let mut service_routes: Vec<_> = app_service_endpoints(state, sandbox) + .await? .into_iter() .filter(|ep| { ep.sandbox_id == sandbox.object_id() @@ -424,6 +432,31 @@ pub(super) async fn configured_app_routes( Ok(service_routes) } +async fn app_service_endpoints( + state: &ServerState, + sandbox: &Sandbox, +) -> Result, Status> { + let selector = format!("sandbox={}", sandbox.object_name()); + let page = state + .store + .list_message_page::( + ObjectListQuery::WorkspaceSelector { + workspace: sandbox.object_workspace(), + label_selector: &selector, + }, + None, + MAX_APP_SERVICE_ROUTES, + ) + .await + .map_err(|_| Status::unavailable("service declarations unavailable"))?; + if page.next_cursor.is_some() { + return Err(Status::resource_exhausted( + "too many application service declarations", + )); + } + Ok(page.messages) +} + async fn get_service_endpoint( state: &Arc, workspace: &str, @@ -536,6 +569,130 @@ mod tests { state.store.put_message(&sandbox).await.unwrap(); } + async fn app_listener_state() -> Arc { + let mut state = test_server_state().await; + Arc::get_mut(&mut state) + .unwrap() + .config + .supervisor_app_listener = openshell_core::app_listener::AppListenerConfig { + enabled: true, + port: 9443, + domain: "services.example.com".into(), + }; + seed_sandbox(&state, "limited").await; + state + } + + async fn seed_app_services(state: &Arc, count: u32) { + for index in 0..count { + let name = format!("web-{index}"); + state + .store + .put_message(&ServiceEndpoint { + metadata: Some(ObjectMeta { + id: format!("endpoint-{index}"), + name: service_routing::endpoint_key("limited", &name), + workspace: "default".into(), + labels: HashMap::from([("sandbox".into(), "limited".into())]), + ..Default::default() + }), + sandbox_id: "sandbox-limited".into(), + sandbox: "limited".into(), + name, + target_port: 8080, + domain: true, + authorization_mode: ServiceAuthorizationMode::Strip as i32, + }) + .await + .unwrap(); + } + } + + async fn expose_limited( + state: &Arc, + name: &str, + ) -> Result, Status> { + handle_expose_service( + state, + authed_request(ExposeServiceRequest { + sandbox: "limited".into(), + name: name.into(), + target_port: 9090, + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + ..Default::default() + }), + ) + .await + } + + #[tokio::test] + async fn app_listener_rejects_overflow_before_persistence_and_keeps_config_deliverable() { + let state = app_listener_state().await; + seed_app_services(&state, MAX_APP_SERVICE_ROUTES).await; + assert_eq!( + expose_limited(&state, "overflow").await.unwrap_err().code(), + tonic::Code::ResourceExhausted + ); + assert!( + get_service_endpoint(&state, "default", "limited", "overflow") + .await + .unwrap() + .is_none() + ); + // Editing a route at capacity must remain possible, as must unrelated + // policy changes and the complete configuration that delivers them. + expose_limited(&state, "web-0").await.unwrap(); + super::super::policy::handle_update_config( + &state, + authed_request(openshell_core::proto::UpdateConfigRequest { + sandbox: "limited".into(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + policy: Some(openshell_policy::restrictive_default_policy()), + ..Default::default() + }), + ) + .await + .unwrap(); + let sandbox = state + .store + .get_message::("sandbox-limited") + .await + .unwrap() + .unwrap(); + let config = super::super::policy::load_sandbox_config(&state, &sandbox) + .await + .unwrap(); + assert_eq!(config.service_routes.len(), MAX_APP_SERVICE_ROUTES as usize); + assert!(config.policy.is_some()); + assert_eq!(config.service_routes[0].target_port, 9090); + } + + #[tokio::test] + async fn app_listener_concurrent_exposures_cannot_exceed_capacity() { + let state = app_listener_state().await; + seed_app_services(&state, MAX_APP_SERVICE_ROUTES - 1).await; + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let spawn_exposure = |name: &'static str| { + let state = state.clone(); + let barrier = barrier.clone(); + tokio::spawn(async move { + barrier.wait().await; + expose_limited(&state, name).await + }) + }; + let (first, second) = tokio::join!(spawn_exposure("first"), spawn_exposure("second")); + let results = [first.unwrap(), second.unwrap()]; + assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .find_map(|result| result.as_ref().err()) + .unwrap() + .code(), + tonic::Code::ResourceExhausted + ); + } + #[test] fn validates_good_endpoint_name() { validate_endpoint_name("service", "web-api", 28).unwrap(); diff --git a/crates/openshell-supervisor/src/app_listener.rs b/crates/openshell-supervisor/src/app_listener.rs index a6ca598eb0..62f7ad81e6 100644 --- a/crates/openshell-supervisor/src/app_listener.rs +++ b/crates/openshell-supervisor/src/app_listener.rs @@ -140,6 +140,14 @@ pub struct Routes { registration: Option>, sender: watch::Sender, stopped: Arc, + #[cfg(test)] + install_probe: Option>, +} + +#[cfg(test)] +struct InstallProbe { + checked_running: std::sync::Barrier, + resume: std::sync::Barrier, } impl Default for Routes { @@ -148,6 +156,8 @@ impl Default for Routes { sender: watch::channel(Snapshot::default()).0, stopped: Arc::new(AtomicBool::new(false)), registration: None, + #[cfg(test)] + install_probe: None, } } } @@ -177,6 +187,11 @@ impl Routes { if self.stopped.load(Ordering::Acquire) { return; } + #[cfg(test)] + if let Some(probe) = &self.install_probe { + probe.checked_running.wait(); + probe.resume.wait(); + } let mut routes = BTreeMap::new(); if !result.configuration_admitted { self.suspend(); @@ -209,7 +224,7 @@ impl Routes { return; } if let Some(registration) = &self.registration - && !registration.bind(&result.workspace) + && !registration.bind(&result.workspace, &self.stopped) { self.suspend(); return; @@ -359,12 +374,18 @@ struct Registration { } impl Registration { - fn bind(&self, workspace: &str) -> bool { + fn bind(&self, workspace: &str, stopped: &AtomicBool) -> bool { let next = format!("{workspace}--{}", self.sandbox); let mut prefix = self .prefix .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + // Shutdown sets stopped before taking this same prefix lock to remove + // the registration. An install that passed its first check must not + // reinsert the old sender after that removal. + if stopped.load(Ordering::Acquire) { + return false; + } let mut entries = self .router .registrations @@ -1175,6 +1196,42 @@ mod tests { ); } + #[tokio::test] + async fn install_racing_shutdown_cannot_reinsert_a_stopped_instance() { + let f = fixture(); + let probe = Arc::new(InstallProbe { + checked_running: std::sync::Barrier::new(2), + resume: std::sync::Barrier::new(2), + }); + let mut installing_routes = f.routes.clone(); + installing_routes.install_probe = Some(probe.clone()); + let snapshot = installed(vec![f.route.clone()]); + let installer = std::thread::spawn(move || installing_routes.install(&snapshot)); + probe.checked_running.wait(); + // Complete shutdown after install's optimistic stopped check but + // before its registration insertion, rather than relying on timing. + drop(f.handle); + probe.resume.wait(); + installer.join().unwrap(); + assert!( + !f.host + .router() + .registrations + .lock() + .unwrap() + .contains_key("default--sandbox-a") + ); + let replacement = f.host.router().instance("sandbox-a").unwrap(); + replacement.routes.install(&installed(vec![f.route])); + assert!( + replacement + .routes + .resolve(HOST, "default--sandbox-a", "svc.openshell.internal") + .is_some(), + "the stopped instance must not prevent its replacement from serving" + ); + } + #[tokio::test(start_paused = true)] async fn push_freshness_renews_live_routes_but_cannot_revive_rejected_routes() { let routes = Routes::default(); diff --git a/e2e/kubernetes/app-listener/README.md b/e2e/kubernetes/app-listener/README.md index c3c5a65623..02eabec87c 100644 --- a/e2e/kubernetes/app-listener/README.md +++ b/e2e/kubernetes/app-listener/README.md @@ -56,6 +56,7 @@ Run from the repository root, supplying the public application certificate/CA: KUBECONFIG="$PWD/kubeconfig" uv run --no-project python \ e2e/kubernetes/app-listener/verify.py \ --namespace listener-review-shared \ + --topology shared \ --gateway-endpoint https://localhost:29203 \ --forward-gateway service/listener-review \ --app-ca /path/to/app-ca.crt \ diff --git a/e2e/kubernetes/app-listener/run.py b/e2e/kubernetes/app-listener/run.py index 0e0b954349..523f8144ac 100644 --- a/e2e/kubernetes/app-listener/run.py +++ b/e2e/kubernetes/app-listener/run.py @@ -247,6 +247,8 @@ def wait_for_shared_hosts(): str(HERE / "verify.py"), "--namespace", namespace, + "--topology", + args.topology, "--gateway-endpoint", f"https://localhost:{port}", "--forward-gateway", diff --git a/e2e/kubernetes/app-listener/verify.py b/e2e/kubernetes/app-listener/verify.py index e6e80d59c4..e42949f269 100644 --- a/e2e/kubernetes/app-listener/verify.py +++ b/e2e/kubernetes/app-listener/verify.py @@ -22,9 +22,16 @@ HERE = Path(__file__).resolve().parent +def check_app_service_count(services, topology): + assert not services, ( + f"{topology} topology expected 0 app Services, found {len(services)}" + ) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--namespace", required=True) + parser.add_argument("--topology", choices=("shared", "dedicated"), required=True) parser.add_argument("--gateway-endpoint", required=True) parser.add_argument( "--forward-gateway", @@ -504,12 +511,14 @@ def stop_forward(): host = hostname(name) print("PASS unique serving supervisor and named HTTP route", flush=True) services = json.loads(run([*k, "get", "services", "-o", "json"]).stdout) - assert not any( - service["metadata"]["name"].startswith("os-app-") + app_services = [ + service + for service in services["items"] + if service["metadata"]["name"].startswith("os-app-") or service["metadata"].get("labels", {}).get("openshell.ai/component") == "app-service" - for service in services["items"] - ), "listener deployment created app Services" + ] + check_app_service_count(app_services, args.topology) print("PASS Pod-IP ingress without per-sandbox app Services", flush=True) assert json.loads(probe("http", ip, hostname(name, "")).stdout)[0] == 200 print("PASS unnamed HTTP route", flush=True) diff --git a/e2e/kubernetes/app-listener/verify_test.py b/e2e/kubernetes/app-listener/verify_test.py new file mode 100644 index 0000000000..0f9f5e7b4e --- /dev/null +++ b/e2e/kubernetes/app-listener/verify_test.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Ensure listener E2E rejects app Services in both supervisor topologies.""" + +import unittest + +from verify import check_app_service_count + + +class AppServiceCountTest(unittest.TestCase): + def test_both_topologies_reject_app_services(self): + for topology in ("dedicated", "shared"): + for services in ([{}], [{}, {}]): + with ( + self.subTest(topology=topology, count=len(services)), + self.assertRaisesRegex(AssertionError, "expected 0"), + ): + check_app_service_count(services, topology) + + def test_both_topologies_accept_no_app_services(self): + for topology in ("dedicated", "shared"): + with self.subTest(topology=topology): + check_app_service_count([], topology) + + +if __name__ == "__main__": + unittest.main() diff --git a/tasks/test.toml b/tasks/test.toml index ea4abc8bd9..6541b1c11f 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -288,7 +288,10 @@ run = "e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:app-listener"] description = "Deploy an isolated Helm release and verify application ingress, isolation, freshness, and supervisor ownership" -run = "uv run --no-project python e2e/kubernetes/app-listener/run.py" +run = [ + "uv run --no-project python -m unittest discover -s e2e/kubernetes/app-listener -p '*_test.py'", + "uv run --no-project python e2e/kubernetes/app-listener/run.py", +] ["e2e:kubernetes:shared-supervisor"] description = "Run Kubernetes e2e with a one-member shared supervisor pool for the gateway's sandboxes (gateway supervisor topology)" From e4990bbd94ca30bd51150b8e611e50196310b002 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Fri, 30 Oct 2026 13:23:51 -0700 Subject: [PATCH 4/5] test(supervisor): cover application streams on config rejection Verify rejected application configuration closes existing WebSockets without affecting another sandbox, and accepted recovery cannot revive old streams. Document raw SDK message migration and shared-owner selection before discovery. Signed-off-by: Taylor Mutch --- crates/openshell-sdk/README.md | 13 ++++++ .../openshell-supervisor/src/app_listener.rs | 42 ++++++++++++++++++- e2e/kubernetes/app-listener/README.md | 12 ++++++ sdk/typescript/README.md | 15 +++++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/crates/openshell-sdk/README.md b/crates/openshell-sdk/README.md index 4df4cde8a4..001b2e9249 100644 --- a/crates/openshell-sdk/README.md +++ b/crates/openshell-sdk/README.md @@ -15,6 +15,19 @@ gateway-name resolution. surface doesn't yet cover (providers, policy, logs, settings, SSH, forwarding). +## Raw configuration message migration + +`SandboxConfigSnapshot` and `GetSandboxConfigResponse` now include the repeated +`service_routes` field. This is an additive protobuf wire change; ordinary RPC +callers can continue reading responses. It describes the supervisor's declared +application ingress allowlist, not service discovery. An empty list installs no +application routes. + +Code constructing complete raw Rust message literals must add +`service_routes: Vec::new()` or use `..Default::default()` for unspecified fields. +Exhaustive destructuring patterns must include `service_routes` or end with `..`. +The curated SDK API is unchanged. + ## Auth and refresh The curated surface drives OIDC refresh automatically: proactively before a diff --git a/crates/openshell-supervisor/src/app_listener.rs b/crates/openshell-supervisor/src/app_listener.rs index 62f7ad81e6..a2e95c38ba 100644 --- a/crates/openshell-supervisor/src/app_listener.rs +++ b/crates/openshell-supervisor/src/app_listener.rs @@ -1400,7 +1400,11 @@ mod tests { tokio_tungstenite::WebSocketStream>; async fn websocket(f: &Fixture, service: &str) -> TestWebSocket { - let host = format!("default--sandbox-a--{service}.svc.openshell.internal"); + sandbox_websocket(f, "sandbox-a", service).await + } + + async fn sandbox_websocket(f: &Fixture, sandbox: &str, service: &str) -> TestWebSocket { + let host = format!("default--{sandbox}--{service}.svc.openshell.internal"); let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[f.address]) .await .unwrap(); @@ -1434,6 +1438,42 @@ mod tests { assert!(next.is_none() || next.unwrap().is_err()); } + #[tokio::test] + async fn rejected_configuration_closes_streams_without_affecting_another_sandbox() { + let f = fixture(); + let second = f.host.router().instance("sandbox-b").unwrap(); + second.routes.install(&installed(vec![SandboxServiceRoute { + endpoint_id: "endpoint-b".into(), + ..f.route.clone() + }])); + let second_connector = Arc::new(Connector { + calls: AtomicUsize::new(0), + }); + let _second_handle = second.start(rpc_test::loopback(second_connector)); + let mut rejected_ws = websocket(&f, "web").await; + let mut unaffected_ws = sandbox_websocket(&f, "sandbox-b", "web").await; + echo(&mut rejected_ws, "before rejection").await; + echo(&mut unaffected_ws, "peer before rejection").await; + + let mut rejected = installed(vec![f.route.clone()]); + rejected.configuration_admitted = false; + rejected.configuration_error = "candidate configuration rejected".into(); + f.routes.install(&rejected); + // A live push session must not renew rejected configuration. + f.routes.renew(); + assert!(request(&f, HOST, HOST, "").await.is_empty()); + echo(&mut unaffected_ws, "peer survives rejection").await; + + // Restore before reading the old stream: even coalesced updates must + // revoke its original token, while new connections can recover. + f.routes.install(&installed(vec![f.route.clone()])); + closed(&mut rejected_ws).await; + let mut recovered_ws = websocket(&f, "web").await; + echo(&mut recovered_ws, "after accepted configuration").await; + echo(&mut unaffected_ws, "peer survives recovery").await; + assert_eq!(f.connector.calls.load(Ordering::SeqCst), 2); + } + #[tokio::test] async fn service_edits_close_only_affected_websockets() { let f = fixture(); diff --git a/e2e/kubernetes/app-listener/README.md b/e2e/kubernetes/app-listener/README.md index 02eabec87c..24b64e3236 100644 --- a/e2e/kubernetes/app-listener/README.md +++ b/e2e/kubernetes/app-listener/README.md @@ -99,6 +99,13 @@ The lower-level runner leaves the Helm release and TLS Secret in place. Use check an already disabled release. Configuration rejection and the additional deployment cleanup scenarios remain in the manual checklist below. +The supervisor Rust test +`rejected_configuration_closes_streams_without_affecting_another_sandbox` +automates rejection at the listener configuration boundary with real TLS +WebSockets on a shared listener. It checks stream closure, isolation from another +sandbox, and recovery without reviving old streams. Injecting a rejection through +the deployed gateway remains a manual Kubernetes scenario. + ## Manual deployment and additional scenarios Before deployment, provide an operator-owned `openshell-app-tls` Secret in the @@ -123,6 +130,11 @@ arbitrary member when more than one is present. Check ownership through supervisor admission/release logs in this stage; automated owner selection belongs to the later discovery API. +This manual selection validates the listener independently of the later RFC 0017 +discovery and routing integrations. A Service balancing across all shared +supervisors cannot select the owner. After takeover, repeat owner selection and +update the test destination; discovery consumers will automate that step. + From a permitted client Pod, mount only the public app certificate/CA and run: ```shell diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index c92cfc1634..8b64c9768d 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -310,6 +310,21 @@ const status = await client.raw.getSandboxPolicyStatus({ The raw layer returns the generated wire messages verbatim, preserving proto distinctions (an omitted optional versus an explicitly empty map) that the curated types may smooth over. As curated sub-clients land, prefer them; `raw` stays as the always-available floor. +### Raw configuration message migration + +`SandboxConfigSnapshot` and `GetSandboxConfigResponse` now include a +`serviceRoutes` array. The protobuf wire change is additive, and ordinary RPC +callers continue to receive decoded messages. This field is the supervisor's +declared application ingress allowlist, not service discovery; an empty array +installs no application routes. + +Complete object literals typed as either generated message must add +`serviceRoutes: []`. Alternatively, use `create(SandboxConfigSnapshotSchema, init)` +or `create(GetSandboxConfigResponseSchema, init)` from `@bufbuild/protobuf`, with +the corresponding schema imported from `@nvidia/openshell-sdk/raw`. Schema-based +initialization supplies the empty array when omitted. The curated SDK API is +unchanged. + ## Boundaries The SDK ships primitives, not the CLI's terminal experience. Some things are intentionally out of scope: From 6925a44d6b3f1bf684578d8d71b9992c1d68ac47 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Fri, 30 Oct 2026 15:56:59 -0700 Subject: [PATCH 5/5] fix(app-listener): fix standalone lint and capture CNI setup failures Signed-off-by: Taylor Mutch --- .agents/skills/watch-github-actions/SKILL.md | 6 ++++++ .github/actions/setup-e2e-kind/action.yml | 18 ++++++++++++++++++ .../src/sandbox_runtime.rs | 1 - 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.agents/skills/watch-github-actions/SKILL.md b/.agents/skills/watch-github-actions/SKILL.md index 0a4a946320..c1beed572a 100644 --- a/.agents/skills/watch-github-actions/SKILL.md +++ b/.agents/skills/watch-github-actions/SKILL.md @@ -125,6 +125,12 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId ## View Job Logs +For Kubernetes app-listener E2E failures during kind setup, inspect the uploaded +`results/app-listener/cni-setup/` evidence. It includes node and Calico pod +descriptions, events, and current/previous container logs. A CNI rollout timeout +occurs before the listener tests run; use these diagnostics to distinguish image +pulls, init failures, and readiness failures. + For tmachine Kubernetes conformance failures, inspect the `tmachine-diagnostics-*` artifact. The K3s report includes Agent Sandbox controller state, leader leases, events, and current and previous controller logs. The conformance playbook checks reconciliation after guest boot with a temporary scheduling-gated Sandbox probe before running tests. Distinguish a failed controller probe from a failed conformance scenario; gateway connectivity and controller Deployment availability do not prove reconciliation works. For the `e2e-suites` lane (`.github/workflows/e2e-suites.yml`), the last lines of the job log give the runner's result: PASS, FAIL (exit 1) after the end of the nextest log (which names the failed tests), or ERROR (exit 2) with the step that broke. FAIL and ERROR also appear as annotations on the run's summary page. Then inspect the `e2e-results----` artifact, which holds `junit.xml`, `nextest.log` and, after a failure, the guest's `diagnostics/`. diff --git a/.github/actions/setup-e2e-kind/action.yml b/.github/actions/setup-e2e-kind/action.yml index 285f0eb145..00e8bad876 100644 --- a/.github/actions/setup-e2e-kind/action.yml +++ b/.github/actions/setup-e2e-kind/action.yml @@ -80,6 +80,24 @@ runs: CLUSTER_NAME: ${{ inputs.cluster-name }} run: | set -euo pipefail + collect_cni_diagnostics() { + failure_status=$? + set +e + diagnostics="$GITHUB_WORKSPACE/results/app-listener/cni-setup" + mkdir -p "$diagnostics" + kubectl --context "kind-$CLUSTER_NAME" get nodes -o wide > "$diagnostics/nodes.txt" 2>&1 + kubectl --context "kind-$CLUSTER_NAME" describe nodes > "$diagnostics/node-details.txt" 2>&1 + kubectl --context "kind-$CLUSTER_NAME" -n kube-system get pods -o wide > "$diagnostics/pods.txt" 2>&1 + kubectl --context "kind-$CLUSTER_NAME" -n kube-system get events --sort-by=.lastTimestamp > "$diagnostics/events.txt" 2>&1 + for component in calico-node calico-kube-controllers; do + kubectl --context "kind-$CLUSTER_NAME" -n kube-system describe pods -l "k8s-app=$component" > "$diagnostics/$component-details.txt" 2>&1 + kubectl --context "kind-$CLUSTER_NAME" -n kube-system logs -l "k8s-app=$component" --all-containers --prefix --tail=200 > "$diagnostics/$component.log" 2>&1 + kubectl --context "kind-$CLUSTER_NAME" -n kube-system logs -l "k8s-app=$component" --all-containers --prefix --tail=200 --previous > "$diagnostics/$component-previous.log" 2>&1 + done + cat "$diagnostics/pods.txt" "$diagnostics/events.txt" + exit "$failure_status" + } + trap collect_cni_diagnostics ERR curl -fsSL https://raw.githubusercontent.com/projectcalico/calico/v3.30.3/manifests/calico.yaml \ -o "$RUNNER_TEMP/calico.yaml" sed 's@192.168.0.0/16@10.244.0.0/16@g' "$RUNNER_TEMP/calico.yaml" > "$RUNNER_TEMP/calico-kind.yaml" diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 3ae1d837fc..39876b3d84 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -1411,7 +1411,6 @@ mod tests { enabled, ..Default::default() }, - ..Default::default() }, ); let spec = pod.spec.expect("Pod spec");