diff --git a/crates/core/src/observability/plugin_component.rs b/crates/core/src/observability/plugin_component.rs index f90de36f5..34cf59429 100644 --- a/crates/core/src/observability/plugin_component.rs +++ b/crates/core/src/observability/plugin_component.rs @@ -187,6 +187,9 @@ impl Default for AtofSectionConfig { #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] pub struct AtofEndpointSectionConfig { + /// Optional stable name used by other components to reference this endpoint. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, /// Endpoint URL. pub url: String, /// Transport: `http_post`, `websocket`, or `ndjson`. @@ -1790,7 +1793,40 @@ fn validate_atof_values( "ATOF mode must be 'append' or 'overwrite'".to_string(), ); } + let mut endpoint_names = HashSet::new(); for (index, endpoint) in section.endpoints.iter().enumerate() { + if let Some(name) = endpoint.name.as_deref() { + if name.trim().is_empty() { + push_policy_diag( + diagnostics, + policy.unsupported_value, + "observability.unsupported_value", + Some("atof".to_string()), + Some(format!("endpoints[{index}].name")), + format!("ATOF endpoints[{index}].name must be non-empty"), + ); + } else if name != name.trim() { + push_policy_diag( + diagnostics, + policy.unsupported_value, + "observability.unsupported_value", + Some("atof".to_string()), + Some(format!("endpoints[{index}].name")), + format!( + "ATOF endpoints[{index}].name must not have leading or trailing whitespace" + ), + ); + } else if !endpoint_names.insert(name) { + push_policy_diag( + diagnostics, + policy.unsupported_value, + "observability.unsupported_value", + Some("atof".to_string()), + Some(format!("endpoints[{index}].name")), + format!("ATOF endpoint name {name:?} must be unique"), + ); + } + } validate_atof_endpoint_values(diagnostics, policy, index, endpoint); } } diff --git a/crates/core/tests/unit/observability/plugin_component_tests.rs b/crates/core/tests/unit/observability/plugin_component_tests.rs index 318b1b417..18c7393fb 100644 --- a/crates/core/tests/unit/observability/plugin_component_tests.rs +++ b/crates/core/tests/unit/observability/plugin_component_tests.rs @@ -205,9 +205,10 @@ fn default_config_and_component_conversion_cover_public_shape() { assert!(atof.filename.is_none()); let parsed_atof: AtofSectionConfig = serde_json::from_value(json!({ - "endpoints": [{"url": "http://localhost/events"}] + "endpoints": [{"name": "switchyard", "url": "http://localhost/events"}] })) .unwrap(); + assert_eq!(parsed_atof.endpoints[0].name.as_deref(), Some("switchyard")); assert_eq!(parsed_atof.endpoints[0].transport, "http_post"); assert_eq!(parsed_atof.endpoints[0].field_name_policy, "preserve"); @@ -305,6 +306,7 @@ fn schema_contains_every_supported_observability_option() { "filename", "mode", "endpoints", + "name", "agent_name", "agent_version", "model_name", @@ -570,7 +572,10 @@ fn atof_endpoint_validation_rejects_bad_values() { {"url": "http://localhost/events", "transport": "bogus"}, {"url": "http://localhost/events", "transport": "ndjson", "timeout_millis": 0}, {"url": "not a url", "transport": "http_post"}, - {"url": "http://localhost/events", "transport": "http_post", "field_name_policy": "bogus"} + {"url": "http://localhost/events", "transport": "http_post", "field_name_policy": "bogus"}, + {"name": "switchyard", "url": "http://localhost/first"}, + {"name": "switchyard", "url": "http://localhost/second"}, + {"name": " ", "url": "http://localhost/blank"} ] } }))); @@ -607,6 +612,18 @@ fn atof_endpoint_validation_rejects_bad_values() { .iter() .any(|diag| { diag.field.as_deref() == Some("endpoints[4].field_name_policy") }) ); + assert!( + report + .diagnostics + .iter() + .any(|diag| { diag.field.as_deref() == Some("endpoints[6].name") }) + ); + assert!( + report + .diagnostics + .iter() + .any(|diag| { diag.field.as_deref() == Some("endpoints[7].name") }) + ); } #[test] @@ -616,6 +633,7 @@ fn build_atof_endpoint_config_maps_headers_timeout_and_rejects_transport() { let config = build_atof_endpoint_config( 2, AtofEndpointSectionConfig { + name: Some("switchyard".into()), url: "ws://127.0.0.1:47632/events".into(), transport: "websocket".into(), headers: headers.clone(), @@ -648,6 +666,7 @@ fn build_atof_endpoint_config_maps_headers_timeout_and_rejects_transport() { let error = build_atof_endpoint_config( 3, AtofEndpointSectionConfig { + name: None, url: "http://127.0.0.1:47632/events".into(), transport: "smtp".into(), headers: std::collections::HashMap::new(), @@ -662,6 +681,7 @@ fn build_atof_endpoint_config_maps_headers_timeout_and_rejects_transport() { let error = build_atof_endpoint_config( 4, AtofEndpointSectionConfig { + name: None, url: "http://127.0.0.1:47632/events".into(), transport: "http_post".into(), headers: std::collections::HashMap::new(), diff --git a/crates/switchyard/README.md b/crates/switchyard/README.md index 82778f37e..b182bda3d 100644 --- a/crates/switchyard/README.md +++ b/crates/switchyard/README.md @@ -19,7 +19,7 @@ This crate is intentionally not published to crates.io yet. Build it from the NeMo Relay source checkout with the optional CLI feature while the Switchyard Decision API contract and service/library boundary are still evolving. -## Why use it? +## Why Use It? - **Route through Switchyard decisions**: Select an exact Relay-owned target using a versioned Decision API contract. @@ -31,7 +31,7 @@ Decision API contract and service/library boundary are still evolving. - **Support staged rollout**: Run in enforce or observe-only mode with explicit target bindings and protocol defaults. -## What you get +## What You Get - `SwitchyardConfig`: the typed plugin configuration contract. - `SwitchyardRuntime`: buffered and streaming routing intercepts. @@ -43,7 +43,7 @@ Decision API contract and service/library boundary are still evolving. - Switchyard-owned protocol translation through the pinned `switchyard-translation` dependency. -## Source build +## Source Build The crate is a private workspace member and must be built from source: @@ -56,7 +56,7 @@ The resulting CLI includes the Switchyard component only when the `switchyard` feature is enabled. A default Relay build does not include this experimental integration. -## Runtime boundary +## Runtime Boundary The current integration calls Switchyard's HTTP Decision API at runtime. Relay does not start or supervise the Switchyard service. For ATOF-backed profiles, @@ -70,10 +70,10 @@ the pinned topic-branch commit, local configuration, compatibility smoke test, and trajectory workflow. Translation is already in-process through Switchyard's Rust translation -library. A future in-process DecisionProvider may replace the HTTP Decision API +library. A future in-process DecisionProvider can replace the HTTP Decision API call without changing the Relay-owned dispatch and observability boundary. -## Configuration and registration +## Configuration and Registration The CLI registers the component when built with `--features switchyard` and accepts a `[[components]]` entry with `kind = "switchyard"`. A minimal @@ -98,9 +98,11 @@ anthropic_messages = "my-anthropic-target" ``` For ATOF-backed profiles, configure an enabled Relay ATOF HTTP exporter that -targets the Switchyard `/v1/atof/events` endpoint and use environment-referenced -authentication headers. Keep provider and Decision API credentials outside -tracked configuration files. +has a unique `name`, targets the Switchyard ingestion URL, and uses +environment-referenced authentication headers. Set `atof_endpoint_name` in the +Switchyard component to that name. Local ATOF JSONL output alone does not +populate the Switchyard accumulator. Keep provider and Decision API credentials +outside tracked configuration files. ## Documentation diff --git a/crates/switchyard/src/component.rs b/crates/switchyard/src/component.rs index ab48e258b..f7d9e364f 100644 --- a/crates/switchyard/src/component.rs +++ b/crates/switchyard/src/component.rs @@ -217,9 +217,9 @@ pub struct SwitchyardConfig { pub targets: BTreeMap, /// Trusted per-protocol fallbacks. pub default_targets: ProtocolDefaults, - /// Optional explicit ATOF endpoint used by CLI cross-validation. + /// Named observability ATOF endpoint used by history-backed profiles. #[serde(default)] - pub atof_endpoint_url: Option, + pub atof_endpoint_name: Option, } impl Default for SwitchyardConfig { @@ -244,7 +244,7 @@ impl Default for SwitchyardConfig { openai_responses: String::new(), anthropic_messages: String::new(), }, - atof_endpoint_url: None, + atof_endpoint_name: None, } } } @@ -269,7 +269,7 @@ nemo_relay::editor_config! { enabled_inbound_profiles => { label: "Enabled inbound profiles", kind: Json }, targets => { label: "Backend target bindings", kind: Json }, default_targets => { label: "Trusted protocol defaults", kind: Json }, - atof_endpoint_url => { label: "ATOF endpoint URL", kind: String, optional: true } + atof_endpoint_name => { label: "ATOF endpoint name", kind: String, optional: true } } } @@ -401,10 +401,9 @@ pub fn validate_switchyard_atof_configuration(config: &PluginConfig) -> Result<( if switchyard.context_mode != ContextMode::AtofRequired { return Ok(()); } - let required_url = match &switchyard.atof_endpoint_url { - Some(url) => url.clone(), - None => derived_atof_url(&switchyard.decision_api_url)?, - }; + let required_name = switchyard.atof_endpoint_name.as_deref().ok_or_else(|| { + "atof_required Switchyard profiles require atof_endpoint_name".to_string() + })?; let observability = config .components .iter() @@ -419,49 +418,55 @@ pub fn validate_switchyard_atof_configuration(config: &PluginConfig) -> Result<( .ok_or_else(|| { "atof_required Switchyard profiles require an enabled ATOF endpoint".to_string() })?; - let endpoint = endpoints + let matching_endpoints = endpoints .iter() - .find(|endpoint| { - endpoint.get("url").and_then(Json::as_str) == Some(required_url.as_str()) - && endpoint - .get("transport") - .and_then(Json::as_str) - .unwrap_or("http_post") - == "http_post" - }) - .ok_or_else(|| { - format!("atof_required Switchyard profile requires HTTP ATOF endpoint {required_url}") - })?; + .filter(|endpoint| endpoint.get("name").and_then(Json::as_str) == Some(required_name)) + .collect::>(); + let endpoint = match matching_endpoints.as_slice() { + [endpoint] => *endpoint, + [] => { + return Err(format!( + "atof_required Switchyard profile requires named ATOF endpoint {required_name:?}" + )); + } + _ => { + return Err(format!( + "ATOF endpoint name {required_name:?} must resolve to exactly one endpoint" + )); + } + }; + if endpoint + .get("transport") + .and_then(Json::as_str) + .unwrap_or("http_post") + != "http_post" + { + return Err(format!( + "Switchyard ATOF endpoint {required_name:?} must use transport = http_post" + )); + } if endpoint .get("field_name_policy") .and_then(Json::as_str) .unwrap_or("preserve") != "preserve" { - return Err("Switchyard ATOF endpoint must use field_name_policy = preserve".into()); + return Err(format!( + "Switchyard ATOF endpoint {required_name:?} must use field_name_policy = preserve" + )); } if endpoint .get("header_env") .and_then(Json::as_object) .is_none_or(Map::is_empty) { - return Err( - "Switchyard ATOF endpoint authentication must use at least one environment-referenced header" - .into(), - ); + return Err(format!( + "Switchyard ATOF endpoint {required_name:?} authentication must use at least one environment-referenced header" + )); } Ok(()) } -fn derived_atof_url(decision_api_url: &str) -> Result { - let mut url = reqwest::Url::parse(decision_api_url) - .map_err(|error| format!("decision_api_url is invalid: {error}"))?; - url.set_path("/v1/atof/events"); - url.set_query(None); - url.set_fragment(None); - Ok(url.to_string().trim_end_matches('/').to_string()) -} - fn parse_config(config: &Map) -> Result { serde_json::from_value(Json::Object(config.clone())) .map_err(|error| format!("invalid Switchyard plugin config: {error}")) @@ -1237,6 +1242,18 @@ fn validate_config(config: &SwitchyardConfig) -> Result<(), String> { if config.recent_message_count == 0 { return Err("recent_message_count must be greater than zero".into()); } + let atof_endpoint_name = config.atof_endpoint_name.as_deref(); + if config.context_mode == ContextMode::AtofRequired && atof_endpoint_name.is_none() { + return Err("atof_required Switchyard profiles require atof_endpoint_name".into()); + } + if let Some(name) = atof_endpoint_name { + if name.trim().is_empty() { + return Err("atof_endpoint_name must be non-empty when configured".into()); + } + if name != name.trim() { + return Err("atof_endpoint_name must not have leading or trailing whitespace".into()); + } + } let url = reqwest::Url::parse(&config.decision_api_url) .map_err(|error| format!("decision_api_url is invalid: {error}"))?; if !matches!(url.scheme(), "http" | "https") { diff --git a/crates/switchyard/tests/unit/component_tests.rs b/crates/switchyard/tests/unit/component_tests.rs index a6e7d5e92..d02d37802 100644 --- a/crates/switchyard/tests/unit/component_tests.rs +++ b/crates/switchyard/tests/unit/component_tests.rs @@ -254,6 +254,12 @@ fn configuration_validation_rejects_unsafe_or_ambiguous_bindings() { let mut candidate = config(url.clone()); candidate.recent_message_count = 0; assert_invalid(candidate, "recent_message_count"); + let mut candidate = config(url.clone()); + candidate.context_mode = ContextMode::AtofRequired; + assert_invalid(candidate, "atof_endpoint_name"); + let mut candidate = config(url.clone()); + candidate.atof_endpoint_name = Some(" switchyard ".into()); + assert_invalid(candidate, "leading or trailing whitespace"); let candidate = config("file:///tmp/decision".into()); assert_invalid(candidate, "must use http or https"); let mut candidate = config(url.clone()); @@ -340,7 +346,7 @@ fn atof_cross_component_validation_reports_each_activation_mismatch() { let mut switchyard = config("http://switchyard.test/v1/routing/decision".into()); switchyard.context_mode = ContextMode::AtofRequired; - switchyard.atof_endpoint_url = Some("http://events.test/v1/atof/events".into()); + switchyard.atof_endpoint_name = Some("switchyard".into()); let mut plugin_config = PluginConfig { components: vec![switchyard.into()], ..PluginConfig::default() @@ -349,7 +355,8 @@ fn atof_cross_component_validation_reports_each_activation_mismatch() { kind: "observability".into(), enabled: true, config: json!({"atof": {"enabled": true, "endpoints": [{ - "url": "http://wrong.test/v1/atof/events", + "name": "other", + "url": "http://events.test/v1/atof/events", "header_env": {"authorization": "TOKEN"} }]}}) .as_object() @@ -359,11 +366,31 @@ fn atof_cross_component_validation_reports_each_activation_mismatch() { assert!( validate_switchyard_atof_configuration(&plugin_config) .unwrap_err() - .contains("requires HTTP ATOF endpoint") + .contains("requires named ATOF endpoint") ); - plugin_config.components[1].config["atof"]["endpoints"][0]["url"] = - json!("http://events.test/v1/atof/events"); + plugin_config.components[1].config["atof"]["endpoints"][0]["name"] = json!("switchyard"); + plugin_config.components[1].config["atof"]["endpoints"][0]["transport"] = json!("websocket"); + assert!( + validate_switchyard_atof_configuration(&plugin_config) + .unwrap_err() + .contains("transport = http_post") + ); + plugin_config.components[1].config["atof"]["endpoints"][0]["transport"] = json!("http_post"); + let duplicate = plugin_config.components[1].config["atof"]["endpoints"][0].clone(); + plugin_config.components[1].config["atof"]["endpoints"] + .as_array_mut() + .unwrap() + .push(duplicate); + assert!( + validate_switchyard_atof_configuration(&plugin_config) + .unwrap_err() + .contains("exactly one endpoint") + ); + plugin_config.components[1].config["atof"]["endpoints"] + .as_array_mut() + .unwrap() + .pop(); plugin_config.components[1].config["atof"]["endpoints"][0]["field_name_policy"] = json!("snake_case"); assert!( @@ -677,6 +704,7 @@ fn identity_policy_requires_stable_request_scope_only_for_atof_profiles() { assert_eq!(synthetic.identity.quality, "synthetic"); config.context_mode = ContextMode::AtofRequired; + config.atof_endpoint_name = Some("switchyard".into()); let atof_runtime = SwitchyardRuntime::new(config).unwrap(); assert!( atof_runtime @@ -808,6 +836,7 @@ fn routing_decision_mark_has_canonical_shape_and_mirrored_identity() { fn atof_required_cross_component_validation_is_context_sensitive() { let mut switchyard = config("http://switchyard.test:8080/v1/routing/decision".into()); switchyard.context_mode = ContextMode::AtofRequired; + switchyard.atof_endpoint_name = Some("switchyard".into()); let mut plugin_config = PluginConfig { components: vec![switchyard.into()], ..PluginConfig::default() @@ -819,6 +848,7 @@ fn atof_required_cross_component_validation_is_context_sensitive() { config: json!({"atof": { "enabled": true, "endpoints": [{ + "name": "switchyard", "url": "http://switchyard.test:8080/v1/atof/events", "transport": "http_post", "field_name_policy": "preserve", diff --git a/docs/configure-plugins/about.mdx b/docs/configure-plugins/about.mdx index 3612a0b5e..054e11189 100644 --- a/docs/configure-plugins/about.mdx +++ b/docs/configure-plugins/about.mdx @@ -33,6 +33,9 @@ entries in `plugins.toml`. Guardrails-backed policy checks. - [PII Redaction](/configure-plugins/pii-redaction/about) sanitizes sensitive data in observability payloads. +- [Switchyard (Experimental)](/configure-plugins/switchyard/about) routes + requests using decisions from a separately running Switchyard Decision API + service. - [Model Pricing](/configure-plugins/model-pricing) configures catalog sources for cost estimates on managed LLM responses. diff --git a/docs/configure-plugins/observability/atof.mdx b/docs/configure-plugins/observability/atof.mdx index d0103354e..1a1c42da4 100644 --- a/docs/configure-plugins/observability/atof.mdx +++ b/docs/configure-plugins/observability/atof.mdx @@ -37,6 +37,7 @@ filename = "events.jsonl" mode = "overwrite" [[components.config.atof.endpoints]] +name = "archive" url = "http://localhost:8080/events" transport = "http_post" timeout_millis = 3000 @@ -69,12 +70,18 @@ The following table describes each streaming endpoint: | Field | Default | Notes | |---|---|---| +| `name` | None | Optional unique name that another component can use to reference this endpoint. Names must be non-empty and cannot have leading or trailing whitespace. | | `url` | Required | Endpoint URL. | | `transport` | `http_post` | `http_post`, `websocket`, or `ndjson`. | | `headers` | `{}` | String-to-string headers for requests or handshakes. | +| `header_env` | `{}` | Map of header names to environment-variable names. Referenced values must be present and non-blank. | | `timeout_millis` | `3000` | Per-endpoint timeout. Must be greater than `0`. | | `field_name_policy` | `preserve` | Field-name handling before Relay sends endpoint events. Accepted values are `preserve` and `replace_dots`. | +Do not define the same header in both `headers` and `header_env`. Use +`header_env` for secrets so configuration stores the environment-variable name +instead of the secret value. + `preserve` sends canonical ATOF field names unchanged. `replace_dots` replaces dots in JSON object keys with underscores recursively. If replacement produces a collision, Relay keeps both values and deterministically appends `_2`, `_3`, @@ -226,9 +233,11 @@ let component = ComponentSpec::new(ObservabilityConfig { filename: Some("events.jsonl".into()), mode: "overwrite".into(), endpoints: vec![AtofEndpointSectionConfig { + name: Some("archive".into()), url: "http://localhost:8080/events".into(), transport: "http_post".into(), headers: Default::default(), + header_env: Default::default(), timeout_millis: 3000, field_name_policy: "replace_dots".into(), }], diff --git a/docs/configure-plugins/observability/configuration.mdx b/docs/configure-plugins/observability/configuration.mdx index 0bb88f45a..a6776c8ba 100644 --- a/docs/configure-plugins/observability/configuration.mdx +++ b/docs/configure-plugins/observability/configuration.mdx @@ -73,6 +73,7 @@ filename = "events.jsonl" mode = "overwrite" [[components.config.atof.endpoints]] +name = "archive" url = "http://localhost:8080/events" transport = "http_post" timeout_millis = 3000 @@ -320,9 +321,11 @@ let component = ComponentSpec::new(ObservabilityConfig { filename: Some("events.jsonl".into()), mode: "overwrite".into(), endpoints: vec![AtofEndpointSectionConfig { + name: Some("archive".into()), url: "http://localhost:8080/events".into(), transport: "http_post".into(), headers: [("authorization".into(), "Bearer ".into())].into(), + header_env: Default::default(), timeout_millis: 3000, field_name_policy: "preserve".into(), }], diff --git a/docs/configure-plugins/switchyard/about.mdx b/docs/configure-plugins/switchyard/about.mdx new file mode 100644 index 000000000..201aa9e7f --- /dev/null +++ b/docs/configure-plugins/switchyard/about.mdx @@ -0,0 +1,73 @@ +--- +title: "Switchyard (Experimental)" +sidebar-title: "Switchyard (Experimental)" +description: "Route Relay LLM requests through the experimental Switchyard Decision API integration." +position: 5 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +> **Experimental:** The Switchyard integration is an early-access feature. It +> is not enabled in default Relay builds, its configuration and contracts can +> change, and the current deployment requires a separately running Switchyard +> Decision API service. + +The `nemo-relay-switchyard` plugin uses the +[Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) Decision API to select +backends for supported LLM requests. Relay validates each decision, translates +requests and responses in process through Switchyard's +`switchyard-translation` library, and dispatches directly to the selected +Relay-owned target. Relay also owns provider credentials, retries, trusted +fallbacks, and observability. The separately running Switchyard service owns +routing decisions. + +## Before You Start + +The current integration has two runtime pieces: + +1. A Relay build with the optional `switchyard` feature. +2. A Switchyard service built from the pinned + [`topic/nemo-relay-integration` revision](https://github.com/NVIDIA-NeMo/Switchyard/tree/topic/nemo-relay-integration). + +The current Relay examples pin Switchyard commit +[`8f9db9a6`](https://github.com/NVIDIA-NeMo/Switchyard/commit/8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8). +Treat that revision as part of this experimental compatibility boundary; update +it deliberately when testing a different Switchyard topic commit. + +The plugin does not start or supervise the Switchyard service. Relay does not +require the service to be healthy at startup; unavailable Decision API calls +fail open to the configured same-protocol fallback. Profiles that depend on +ATOF history, including StageRouter, must use +`context_mode = "atof_required"` and name an enabled Relay ATOF HTTP endpoint +with `atof_endpoint_name`. Relay fails startup validation when that endpoint is +missing or invalid. Relay pushes ATOF events to the endpoint; local ATOF JSONL +output alone does not populate Switchyard's accumulator. + +Build the opt-in CLI feature with: + +```bash +cargo build -p nemo-relay-cli --features switchyard +``` + +The default CLI build does not include the plugin or the Switchyard translation +library. + +## Capabilities + +- Enforce or observe-only routing decisions. +- OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages inbound + profiles. +- Buffered and streaming protocol translation through Switchyard's + `switchyard-translation` library. +- Exact Relay-owned backend bindings and per-protocol trusted fallbacks. +- Bounded retries before the first streaming item. +- Canonical routing marks and shared LLM optimization accounting. + +## Pages + +- [Switchyard Configuration](/configure-plugins/switchyard/configuration) + is the complete option and deployment reference. Review its + [experimental limitations](/configure-plugins/switchyard/configuration#experimental-limitations) + before adopting the integration. +- [Switchyard integration examples](https://github.com/NVIDIA/NeMo-Relay/tree/main/examples/switchyard) + documents the pinned service worktree and manual compatibility smoke tests. diff --git a/docs/configure-plugins/switchyard/configuration.mdx b/docs/configure-plugins/switchyard/configuration.mdx new file mode 100644 index 000000000..39e5b82f0 --- /dev/null +++ b/docs/configure-plugins/switchyard/configuration.mdx @@ -0,0 +1,333 @@ +--- +title: "Switchyard Configuration" +sidebar-title: "Configuration" +description: "Configure the experimental Relay-native Switchyard Decision API plugin." +position: 2 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +> **Experimental:** This page describes the current service-based Switchyard +> integration. Its configuration and service boundary can change as Switchyard +> develops an in-process Decision Provider. + +The Switchyard component connects Relay's CLI gateway to a separately running +[Switchyard](https://github.com/NVIDIA-NeMo/Switchyard) Decision API. Relay +owns provider credentials, backend endpoints, dispatch, retries, and trusted +fallbacks. The Switchyard service selects a backend. Relay validates the +decision, translates requests and responses in process through the +`switchyard-translation` library, and dispatches directly to the selected +Relay-owned target. + +## How the Configuration Fits Together + +Configure the integration in four core steps. Profiles that use ATOF history +add the fifth requirement: + +1. Set `decision_api_url` and authentication so Relay can call the Switchyard + Decision API. +2. Select the Switchyard routing profile with `decision_profile_id`. +3. Add a Relay-owned `targets` binding for every Switchyard `backend_id` that + the profile can return. Relay validates the returned backend ID, model, + protocol, and endpoint exactly before dispatch. +4. Choose a trusted same-protocol target for each inbound protocol under + `default_targets`. Relay uses these defaults for observe-only traffic and + fail-open dispatch. +5. For a profile that depends on ATOF history, including StageRouter, set + `context_mode = "atof_required"`, set `atof_endpoint_name`, and configure the + matching observability endpoint. Relay fails startup validation if the + named endpoint is missing or invalid. + +The following is a complete component configuration. The referenced +environment variables must be set before Relay starts. + +```toml +version = 1 + +[[components]] +kind = "switchyard" +enabled = true + +[components.config] +version = 1 +mode = "enforce" +priority = 0 +decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" +decision_profile_id = "stage-router" +request_materialization = "recent_message_window" +context_mode = "atof_required" +atof_endpoint_name = "switchyard" +decision_timeout_millis = 25 +max_retries = 3 +recent_message_count = 8 +enabled_inbound_profiles = ["openai_chat", "openai_responses", "anthropic_messages"] + +[components.config.decision_header_env] +authorization = "SWITCHYARD_AUTHORIZATION" + +[components.config.default_targets] +openai_chat = "openai-default" +openai_responses = "responses-default" +anthropic_messages = "anthropic-default" + +[components.config.targets.openai-default] +model = "provider/efficient-chat" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" + +[components.config.targets.openai-default.header_env] +authorization = "OPENAI_PROVIDER_AUTHORIZATION" + +[components.config.targets.responses-default] +model = "provider/efficient-responses" +protocol = "openai_responses" +endpoint = "/v1/responses" +base_url = "https://provider.example.com" + +[components.config.targets.responses-default.header_env] +authorization = "OPENAI_PROVIDER_AUTHORIZATION" + +[components.config.targets.anthropic-default] +model = "provider/efficient-messages" +protocol = "anthropic_messages" +endpoint = "/v1/messages" +base_url = "https://provider.example.com" + +[components.config.targets.anthropic-default.header_env] +x-api-key = "ANTHROPIC_PROVIDER_API_KEY" + +[components.config.targets.capable-chat] +model = "provider/capable-chat" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" + +[components.config.targets.capable-chat.header_env] +authorization = "OPENAI_PROVIDER_AUTHORIZATION" + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 1 + +[components.config.atof] +enabled = true + +[[components.config.atof.endpoints]] +name = "switchyard" +url = "http://127.0.0.1:4000/v1/atof/events" +transport = "http_post" +field_name_policy = "preserve" + +[components.config.atof.endpoints.header_env] +authorization = "SWITCHYARD_AUTHORIZATION" +``` + +## Decision API and Rollout Options + +| Option | Default | Description and constraints | +| --- | --- | --- | +| `version` | `1` | Component configuration version. | +| `mode` | `"enforce"` | `enforce` applies valid decisions. `observe_only` records the hypothetical decision but dispatches the trusted same-protocol default once. | +| `priority` | `0` | Component execution priority. | +| `decision_api_url` | Required | HTTP or HTTPS URL for the Switchyard Decision API. | +| `decision_profile_id` | Required | Non-empty Switchyard profile identifier sent with each routing request. | +| `decision_timeout_millis` | `25` | Decision API timeout in milliseconds; must be greater than zero. | +| `decision_headers` | Empty | Optional non-sensitive static Decision API headers. | +| `decision_header_env` | Empty | Map of header names to environment-variable names for sensitive Decision API headers. Referenced values must be present and non-blank. | + +Do not define the same header in both a static header map and its environment +map. Keep bearer tokens, API keys, cookies, and other secrets in environment +variables rather than tracked configuration. + +## Request Materialization + +Every Decision API request contains routing identity, inbound protocol, a +request summary, and the attempt number. `request_materialization` controls how +much additional request content is included. + +| Mode | Material sent to Switchyard | +| --- | --- | +| `none` | No `current_request`; use identity, protocol, summary, and attempt metadata only. | +| `summary_only` | No `current_request`; explicitly selects summary-based routing. | +| `latest_user_prompt` | A provider-valid body containing the latest user material plus an explicit prompt field. The request fails open if no user message is available. | +| `recent_message_window` | The most recent normalized messages, bounded by `recent_message_count`, re-encoded for the inbound protocol. | +| `annotated_request` | The final intercepted provider body plus Relay's normalized request representation. | +| `full_body` | The final intercepted provider request body. | + +Choose the least materialization needed by the selected profile. In particular, +`none` and `summary_only` intentionally share the same current wire behavior but +express different configuration intent. + +## Context, Retries, and Inbound Protocols + +| Option | Default | Description and constraints | +| --- | --- | --- | +| `context_mode` | Required | `payload_only` or `atof_required`. See below. | +| `max_retries` | `3` | Provider retries after the initial attempt; maximum `10`. Each retry obtains a new Switchyard decision. | +| `recent_message_count` | `8` | Message limit for `recent_message_window`; must be greater than zero. | +| `enabled_inbound_profiles` | All three supported protocols | Non-empty list containing `openai_chat`, `openai_responses`, and/or `anthropic_messages`. | +| `atof_endpoint_name` | Required for `atof_required` | Name of exactly one endpoint under the observability component's `atof.endpoints` list. | + +`payload_only` supports request-aware routing without an ATOF dependency. +Relay can create request-scoped synthetic identity when stable identity is not +available. + +`atof_required` is for profiles that classify from accumulated ATOF context. It +requires stable session and request identity plus an enabled, authenticated +Relay HTTP ATOF endpoint whose `name` matches `atof_endpoint_name`. The endpoint +specifies the Switchyard ingestion URL, must use `transport = "http_post"`, and +must set `field_name_policy = "preserve"`. Relay rejects missing or duplicate +named endpoints during startup validation. The ATOF exporter's local JSONL file +does not satisfy this delivery requirement by itself. + +## Target Bindings and Trusted Defaults + +| Option | Required fields | Purpose | +| --- | --- | --- | +| `targets.` | `model`, `protocol`, `endpoint`, `base_url` | Binds an exact Switchyard backend ID to a Relay-owned provider destination. | +| `targets..headers` | None | Optional non-sensitive static provider headers. | +| `targets..header_env` | None | Optional provider header names mapped to environment-variable names. Referenced values must be present and non-blank. | +| `default_targets.openai_chat` | Target using `openai_chat` | Trusted fallback for OpenAI Chat Completions requests. | +| `default_targets.openai_responses` | Target using `openai_responses` | Trusted fallback for OpenAI Responses requests. | +| `default_targets.anthropic_messages` | Target using `anthropic_messages` | Trusted fallback for Anthropic Messages requests. | + +Protocol and endpoint pairs are fixed: + +| Protocol | Endpoint | +| --- | --- | +| `openai_chat` | `/v1/chat/completions` | +| `openai_responses` | `/v1/responses` | +| `anthropic_messages` | `/v1/messages` | + +Each protocol in `enabled_inbound_profiles` requires a corresponding +`default_targets` entry. Defaults for disabled protocols can be omitted. Each +configured default must refer to a target with the matching protocol. Duplicate +exact bindings for model, protocol, endpoint, and base URL are rejected. + +## Recommended Recipes + +### Observe-Only Rollout + +Use observe-only mode to verify decisions and routing marks without sending +traffic to the selected target: + +```toml +[components.config] +mode = "observe_only" +context_mode = "payload_only" +request_materialization = "summary_only" +``` + +Relay calls the Decision API and records the hypothetical selection, but sends +the request to the inbound protocol's trusted default once. Routing retries are +not applied. + +### Payload-Only Request-Aware Routing + +Use recent request content for routing without operating the ATOF accumulator: + +```toml +[components.config] +mode = "enforce" +context_mode = "payload_only" +request_materialization = "recent_message_window" +recent_message_count = 8 +``` + +### ATOF-Backed StageRouter Routing + +Use ATOF context when StageRouter should classify from accumulated session +signals: + +```toml +[components.config] +mode = "enforce" +decision_profile_id = "stage-router" +context_mode = "atof_required" +request_materialization = "summary_only" +atof_endpoint_name = "switchyard" + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 1 + +[components.config.atof] +enabled = true + +[[components.config.atof.endpoints]] +name = "switchyard" +url = "http://127.0.0.1:4000/v1/atof/events" +transport = "http_post" +field_name_policy = "preserve" + +[components.config.atof.endpoints.header_env] +authorization = "SWITCHYARD_AUTHORIZATION" +``` + +## Runtime and Fail-Open Behavior + +- Relay validates the decision's backend ID, model, protocol, and endpoint + against its target binding before dispatch. Unknown backends, malformed + decisions, and target drift fail open. +- Decision API errors are not recursively retried. Relay immediately dispatches + the trusted same-protocol default. +- Retryable provider connection, timeout, status, context-window, and + model-unavailable failures can obtain a new decision, bounded by + `max_retries`. +- Non-retryable provider failures use the trusted fallback immediately. After + retry exhaustion, Relay dispatches that fallback once. +- A streaming request can retry only before receiving its first upstream item. + After the stream commits, Relay propagates later failures without another + dispatch. +- Unsupported provider extensions and request translation failures before + provider dispatch fail open to the trusted same-protocol default. Buffered + response translation failures also fail open. After a streaming response + emits its first item, later provider or translation failures propagate + without another dispatch. +- Routing-mark delivery is best-effort and does not change provider results. + +## Experimental Limitations + +- The plugin is source-only, is not independently published, and is excluded + from default Relay builds. Build the CLI with + `cargo build -p nemo-relay-cli --features switchyard`. +- The routing boundary is currently service-based. Relay calls Switchyard's + HTTP Decision API but does not start, supervise, or health-gate the service. + The service can be unavailable at Relay startup; requests fail open while it + remains unavailable. +- ATOF-backed profiles additionally depend on Switchyard's HTTP ingestion and + accumulator runtime, stable request and session identity, and one named, + authenticated Relay ATOF HTTP endpoint. Local JSONL output does not populate + the Switchyard accumulator. +- Provider-protocol translation runs in Relay's process through the + `switchyard-translation` Rust library. Routing decisions and ATOF accumulation + remain out of process. +- The Rust Switchyard component is currently registered only by Relay's CLI + gateway. Hermes native plugins and the LangChain, LangGraph, and Deep Agents + integrations do not load it in process. They receive Switchyard routing only + when their provider traffic is explicitly sent through the Relay gateway. +- The Switchyard examples are manual compatibility and trajectory workflows, + not production deployment orchestration. +- Compatibility is pinned to Switchyard commit + [`8f9db9a6`](https://github.com/NVIDIA-NeMo/Switchyard/commit/8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8) + on `topic/nemo-relay-integration`. Testing another revision is deliberate and + can expose contract drift. +- The Decision API and service/library boundary are experimental and can change + when Switchyard introduces an in-process Decision Provider. +- Unsupported provider extensions, malformed decisions, service failures, + pre-dispatch translation failures, buffered-response translation failures, + and target drift fail open to Relay's trusted same-protocol default. Failures + after a streaming response commits propagate without another dispatch. +- Secrets must remain environment-referenced. The examples generate ephemeral + local credentials and are not credential-management guidance. + +Run `nemo-relay doctor` to validate component configuration before startup. For +the pinned local service and manual compatibility workflow, refer to the +[Switchyard integration examples](https://github.com/NVIDIA/NeMo-Relay/tree/main/examples/switchyard). diff --git a/docs/index.yml b/docs/index.yml index fe797eaa2..fca80db2d 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -40,6 +40,9 @@ navigation: - folder: ./configure-plugins/pii-redaction title: "PII Redaction" title-source: frontmatter + - folder: ./configure-plugins/switchyard + title: "Switchyard (Experimental)" + title-source: frontmatter - page: "Model Pricing" path: ./configure-plugins/model-pricing.mdx slug: model-pricing diff --git a/examples/switchyard/README.md b/examples/switchyard/README.md index 353e8dd19..e9f3e0ee0 100644 --- a/examples/switchyard/README.md +++ b/examples/switchyard/README.md @@ -3,13 +3,13 @@ SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# Switchyard integration examples +# Switchyard Integration Examples These examples exercise the experimental Relay integration with a separately running Switchyard Decision API service and the in-process Switchyard translation library. They are manual, local validation workflows rather than production startup orchestration. -## Required Switchyard revision +## Required Switchyard Revision The scripts default to the latest commit currently pinned for the public topic branch: @@ -18,11 +18,14 @@ https://github.com/NVIDIA-NeMo/Switchyard/tree/topic/nemo-relay-integration 8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 ``` -Create the adjacent checkout from the Relay repository root: +Clone the Switchyard repository next to the Relay checkout, then pin the +required commit: ```bash -git fetch upstream topic/nemo-relay-integration -git worktree add --detach ../Switchyard-topic-nemo-relay-integration \ +git clone --branch topic/nemo-relay-integration \ + https://github.com/NVIDIA-NeMo/Switchyard.git \ + ../Switchyard-topic-nemo-relay-integration +git -C ../Switchyard-topic-nemo-relay-integration checkout --detach \ 8f9db9a6a47f848cdff1d262276ba25a8ae9cbc8 ``` @@ -42,7 +45,7 @@ Run these commands from the root of the NeMo Relay checkout. The Relay CLI's Switchyard support is compile-time optional. The scripts enable the `switchyard` feature automatically; custom builds must pass `--features switchyard`. -### Manual Switchyard compatibility smoke test +### Manual Switchyard Compatibility Smoke Test `run-real-e2e.sh` is a manual compatibility smoke test. It starts the pinned Switchyard server, Relay, and a fake provider, then verifies cold and warm StageRouter decisions, buffered routing, @@ -55,7 +58,7 @@ a successful run. examples/switchyard/run-real-e2e.sh ``` -### Hermes and Ollama trajectory +### Hermes and Ollama Trajectory `run-hermes-ollama-smoke.sh` runs a fixed multi-query trajectory through Hermes, Relay, Ollama, and Switchyard. It requires Docker, Hermes, and the configured local Ollama models. The script @@ -66,7 +69,7 @@ produces ATOF, ATIF, and OTEL artifacts and can leave Phoenix running with examples/switchyard/run-hermes-ollama-smoke.sh ``` -## Configuration files +## Configuration Files - `plugins.toml`: minimal plugin configuration example. - `real-e2e-plugins.toml` and `real-e2e-profiles.yaml`: deterministic fake-provider E2E. @@ -74,18 +77,20 @@ examples/switchyard/run-hermes-ollama-smoke.sh - `fake_upstream.py`: deterministic provider used by the service E2E. - `otel-collector.yaml`: local OTEL artifact export configuration. -## Runtime model +## Runtime Model The scripts launch Switchyard as a separate local process on port `4000`. Relay sends routing requests to `/v1/routing/decision` and, for ATOF-backed profiles, sends events to `/v1/atof/events`. Relay owns provider credentials, target bindings, dispatch, retries, and -fallback behavior; Switchyard owns ATOF accumulation, routing decisions, and provider-protocol -translation. +fallback behavior. Relay executes provider-protocol translation in process through Switchyard's +translation library; the Switchyard service owns ATOF accumulation and routing decisions. The +Switchyard component selects its HTTP ingestion destination by the observability endpoint name, +not by duplicating the endpoint URL. The service is not started automatically by Relay outside these examples. A production deployment must provide a reachable Decision API and configure the Relay plugin with its URL. -## Artifacts and troubleshooting +## Artifacts and Troubleshooting Trajectory scripts write to `artifacts/` by default. Set `SWITCHYARD_TRAJECTORY_DIR` to choose a shareable output directory. On failure, logs are preserved and include the verified Switchyard diff --git a/examples/switchyard/hermes-ollama-plugins.toml b/examples/switchyard/hermes-ollama-plugins.toml index 6a7d128b0..85ce6fa88 100644 --- a/examples/switchyard/hermes-ollama-plugins.toml +++ b/examples/switchyard/hermes-ollama-plugins.toml @@ -13,6 +13,7 @@ decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" decision_profile_id = "hermes-ollama-stage-router" request_materialization = "full_body" context_mode = "atof_required" +atof_endpoint_name = "switchyard" decision_timeout_millis = 120000 max_retries = 3 recent_message_count = 8 @@ -60,6 +61,7 @@ output_directory = "." filename = "trajectory.atof.jsonl" [[components.config.atof.endpoints]] +name = "switchyard" url = "http://127.0.0.1:4000/v1/atof/events" transport = "http_post" field_name_policy = "preserve" diff --git a/examples/switchyard/plugins.toml b/examples/switchyard/plugins.toml index f99d9e4c3..109bdabb9 100644 --- a/examples/switchyard/plugins.toml +++ b/examples/switchyard/plugins.toml @@ -14,6 +14,7 @@ decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" decision_profile_id = "smart-stage-router" request_materialization = "recent_message_window" context_mode = "atof_required" +atof_endpoint_name = "switchyard" decision_timeout_millis = 25 max_retries = 3 recent_message_count = 8 @@ -60,6 +61,7 @@ enabled = true mode = "append" [[components.config.atof.endpoints]] +name = "switchyard" url = "http://127.0.0.1:4000/v1/atof/events" transport = "http_post" field_name_policy = "preserve" diff --git a/examples/switchyard/real-e2e-plugins.toml b/examples/switchyard/real-e2e-plugins.toml index 9c6351a6d..1b5da7063 100644 --- a/examples/switchyard/real-e2e-plugins.toml +++ b/examples/switchyard/real-e2e-plugins.toml @@ -13,6 +13,7 @@ decision_api_url = "http://127.0.0.1:4000/v1/routing/decision" decision_profile_id = "remote-stage-router" request_materialization = "summary_only" context_mode = "atof_required" +atof_endpoint_name = "switchyard" decision_timeout_millis = 1000 max_retries = 3 recent_message_count = 8 @@ -58,6 +59,7 @@ enabled = true mode = "append" [[components.config.atof.endpoints]] +name = "switchyard" url = "http://127.0.0.1:4000/v1/atof/events" transport = "http_post" field_name_policy = "preserve" diff --git a/go/nemo_relay/observability_plugin.go b/go/nemo_relay/observability_plugin.go index c79488919..0d79e2611 100644 --- a/go/nemo_relay/observability_plugin.go +++ b/go/nemo_relay/observability_plugin.go @@ -41,6 +41,7 @@ type ObservabilityAtofConfig struct { // ObservabilityAtofEndpoint configures one streaming destination for raw ATOF events. type ObservabilityAtofEndpoint struct { + Name string `json:"name,omitempty"` URL string `json:"url"` Transport string `json:"transport,omitempty"` Headers map[string]string `json:"headers,omitempty"` diff --git a/go/nemo_relay/observability_plugin_test.go b/go/nemo_relay/observability_plugin_test.go index 7c94c33b9..f09cfd7cf 100644 --- a/go/nemo_relay/observability_plugin_test.go +++ b/go/nemo_relay/observability_plugin_test.go @@ -36,6 +36,7 @@ func TestObservabilityConfigHelpers(t *testing.T) { t.Fatalf("unexpected ATOF defaults: %#v", atof) } atof.Endpoints = []ObservabilityAtofEndpoint{{ + Name: "archive", URL: "http://localhost:8080/events", Transport: "http_post", Headers: map[string]string{"X-Test": "yes"}, @@ -85,15 +86,15 @@ func TestObservabilityConfigHelpers(t *testing.T) { t.Fatalf("expected serialized ATOF endpoints, got %#v", atofConfig) } firstEndpoint, ok := endpoints[0].(map[string]any) - if !ok || firstEndpoint["field_name_policy"] != "replace_dots" { + if !ok || firstEndpoint["name"] != "archive" || firstEndpoint["field_name_policy"] != "replace_dots" { t.Fatalf("expected serialized ATOF endpoint field name policy, got %#v", endpoints) } serialized, err := json.Marshal(wrapped) if err != nil { t.Fatalf("marshal observability component failed: %v", err) } - if !strings.Contains(string(serialized), `"field_name_policy":"replace_dots"`) { - t.Fatalf("expected field_name_policy in serialized component, got %s", serialized) + if !strings.Contains(string(serialized), `"name":"archive"`) || !strings.Contains(string(serialized), `"field_name_policy":"replace_dots"`) { + t.Fatalf("expected named ATOF endpoint in serialized component, got %s", serialized) } assertWrappedAtifStorageConfig(t, wrapped.Config["atif"].(map[string]any)) if wrapped.Config["opentelemetry"].(map[string]any)["mark_projection"] != "tool" {