From fe548d349a6a3b6746d87f90072f403043068adb Mon Sep 17 00:00:00 2001 From: Atish Patel Date: Wed, 29 Jul 2026 09:03:28 -0500 Subject: [PATCH 1/2] feat(agent): route Claude/GPT model families to their native gateway wire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Databricks v2 picks the gateway wire format (OpenAI Responses vs. Anthropic Messages vs. MLflow chat) purely from substrings in the endpoint name, with no family field to rely on. The matcher only knew `gpt-5`/`gpt5` and `claude`, so any Claude endpoint whose catalog name omits the literal "claude" — an alias, a bare `opus-5`, a `goose-opus-5` — silently fell through to the MLflow (OpenAI-wire) path, where Anthropic prompt caching is structurally impossible and the ~10x cache-read discount is lost with no error. Broaden both marker sets: - Claude (Anthropic Messages): `claude`, `opus`, `sonnet`, `haiku`, `mythos`, `fable` — the family names and release code names, so a Claude endpoint reaches the cache-capable route regardless of naming. - OpenAI (Responses): the `gpt` family plus the GPT-5 launch code names `sol`, `luna`, `terra`. OpenAI markers are checked first, preserving the prior `gpt-5`-first precedence. Unrecognised names still fall through to the MLflow chat route. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Atish Patel --- crates/buzz-agent/src/llm.rs | 90 +++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 7 deletions(-) diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index f2359f9ba3..789211f7bf 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -1019,13 +1019,31 @@ fn is_responses_required_error(body: &str) -> bool { || b.contains("use the responses api") } +/// Substrings identifying an OpenAI-shaped model in a Databricks v2 catalog +/// name. Covers the `gpt` family plus Databricks' GPT-5 launch code names +/// (`sol`, `luna`, `terra`). Matched case-insensitively. +const DATABRICKS_V2_OPENAI_MARKERS: &[&str] = &["gpt", "sol", "luna", "terra"]; + +/// Substrings identifying an Anthropic-shaped (Claude) model in a Databricks v2 +/// catalog name. Covers the `claude` prefix, the Claude family names (`opus`, +/// `sonnet`, `haiku`), and the release code names (`mythos`, `fable`). Matched +/// case-insensitively. Getting a Claude model onto this route is what lets it +/// carry a `cache_control` breakpoint — an endpoint whose name matches none of +/// these falls through to the MLflow (OpenAI-wire) path, where Anthropic prompt +/// caching is structurally impossible, so the discount is silently lost. +const DATABRICKS_V2_CLAUDE_MARKERS: &[&str] = + &["claude", "opus", "sonnet", "haiku", "mythos", "fable"]; + fn databricks_v2_route_for_model(model: &str) -> DatabricksV2Route { - // Databricks v2 catalog names currently identify OpenAI-shaped GPT-5 - // models and Anthropic-shaped Claude models by these substrings. + // Databricks v2 catalog names identify the wire format only by these + // substrings — there is no explicit family field on the endpoint. OpenAI + // markers are checked first so a name carrying both resolves to the OpenAI + // wire (preserving the prior `gpt-5`-first behaviour). let lower = model.to_ascii_lowercase(); - if lower.contains("gpt-5") || lower.contains("gpt5") { + let matches_any = |markers: &[&str]| markers.iter().any(|m| lower.contains(m)); + if matches_any(DATABRICKS_V2_OPENAI_MARKERS) { DatabricksV2Route::OpenAiResponses - } else if lower.contains("claude") { + } else if matches_any(DATABRICKS_V2_CLAUDE_MARKERS) { DatabricksV2Route::AnthropicMessages } else { DatabricksV2Route::MlflowChatCompletions @@ -2579,20 +2597,78 @@ mod tests { #[test] fn databricks_v2_routes_by_model_family() { + use DatabricksV2Route::{AnthropicMessages, MlflowChatCompletions, OpenAiResponses}; for (model, route, path) in [ + // OpenAI-shaped: the gpt family plus the GPT-5 code names. ( "databricks-gpt-5-5", - DatabricksV2Route::OpenAiResponses, + OpenAiResponses, + "/ai-gateway/openai/v1/responses", + ), + ("gpt-4o", OpenAiResponses, "/ai-gateway/openai/v1/responses"), + ( + "databricks-gpt-5-6-luna", + OpenAiResponses, + "/ai-gateway/openai/v1/responses", + ), + ( + "databricks-gpt-5-6-sol", + OpenAiResponses, + "/ai-gateway/openai/v1/responses", + ), + ( + "databricks-terra", + OpenAiResponses, "/ai-gateway/openai/v1/responses", ), + // Anthropic-shaped: the claude prefix, the family names, and the + // release code names — each must reach the cache-capable route even + // when the endpoint name omits the literal "claude". ( "databricks-claude-opus-4-7", - DatabricksV2Route::AnthropicMessages, + AnthropicMessages, + "/ai-gateway/anthropic/v1/messages", + ), + ( + "goose-opus-5", + AnthropicMessages, + "/ai-gateway/anthropic/v1/messages", + ), + ( + "databricks-sonnet-5", + AnthropicMessages, "/ai-gateway/anthropic/v1/messages", ), + ( + "databricks-haiku-4-5", + AnthropicMessages, + "/ai-gateway/anthropic/v1/messages", + ), + ( + "databricks-mythos-5", + AnthropicMessages, + "/ai-gateway/anthropic/v1/messages", + ), + ( + "databricks-fable-5", + AnthropicMessages, + "/ai-gateway/anthropic/v1/messages", + ), + // Case-insensitive. + ( + "Databricks-Claude-Opus-5", + AnthropicMessages, + "/ai-gateway/anthropic/v1/messages", + ), + // Unrecognised names still fall through to the MLflow chat route. ( "custom-tool-model", - DatabricksV2Route::MlflowChatCompletions, + MlflowChatCompletions, + "/ai-gateway/mlflow/v1/chat/completions", + ), + ( + "databricks-gemini-3-pro", + MlflowChatCompletions, "/ai-gateway/mlflow/v1/chat/completions", ), ] { From f992d3428129b6e9fe2766f978bd4fb9e73ba703 Mon Sep 17 00:00:00 2001 From: Atish Patel Date: Wed, 29 Jul 2026 09:21:30 -0500 Subject: [PATCH 2/2] fix(agent): match model families by name segment, not substring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review: the family classifier used unrestricted `contains`, so short markers matched unrelated custom aliases. V2 discovery keeps arbitrary endpoint names, so this was reachable, not theoretical: `consolidated-llama` (`sol`) and `terraform-coder` (`terra`) misrouted to OpenAI Responses, and `corpus-reranker`/`octopus-model` (`opus`) to Anthropic Messages — sending each a wire shape its backend can't parse, turning a caching optimization into a hard request/parse failure. Split the name into alphanumeric segments (breaking on any non-alphanumeric delimiter) and match whole segments. Code/family names (`sol`, `luna`, `terra`, `claude`, `opus`, `sonnet`, `haiku`, `mythos`, `fable`) must equal a segment; the `gpt` family matches a segment beginning with `gpt`, preserving the dashless `gpt5` spelling and the `gpt` of a split `gpt-5`. Prefixed real names like `goose-opus-5` still match. Add negative collision cases (`consolidated-llama`, `terraform-coder`, `corpus-reranker`, `octopus-model`) and the `gpt5` spelling to the routing test. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Atish Patel --- crates/buzz-agent/src/llm.rs | 89 ++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index 789211f7bf..f69a963533 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -1019,31 +1019,53 @@ fn is_responses_required_error(body: &str) -> bool { || b.contains("use the responses api") } -/// Substrings identifying an OpenAI-shaped model in a Databricks v2 catalog -/// name. Covers the `gpt` family plus Databricks' GPT-5 launch code names -/// (`sol`, `luna`, `terra`). Matched case-insensitively. -const DATABRICKS_V2_OPENAI_MARKERS: &[&str] = &["gpt", "sol", "luna", "terra"]; - -/// Substrings identifying an Anthropic-shaped (Claude) model in a Databricks v2 -/// catalog name. Covers the `claude` prefix, the Claude family names (`opus`, -/// `sonnet`, `haiku`), and the release code names (`mythos`, `fable`). Matched -/// case-insensitively. Getting a Claude model onto this route is what lets it -/// carry a `cache_control` breakpoint — an endpoint whose name matches none of +/// OpenAI-family code names that appear as their own segment in a Databricks v2 +/// endpoint name (the GPT-5 launch aliases). The `gpt` family itself is matched +/// separately by segment prefix so `gpt`, `gpt5`, and the `gpt` of a split +/// `gpt-5` all qualify. +const DATABRICKS_V2_OPENAI_CODE_NAMES: &[&str] = &["sol", "luna", "terra"]; + +/// Anthropic (Claude) family and release code names that appear as their own +/// segment in a Databricks v2 endpoint name — the `claude` prefix, the family +/// names (`opus`, `sonnet`, `haiku`), and the release code names (`mythos`, +/// `fable`). Getting a Claude model onto the Anthropic Messages route is what +/// lets it carry a `cache_control` breakpoint; an endpoint that matches none of /// these falls through to the MLflow (OpenAI-wire) path, where Anthropic prompt -/// caching is structurally impossible, so the discount is silently lost. -const DATABRICKS_V2_CLAUDE_MARKERS: &[&str] = +/// caching is structurally impossible and the discount is silently lost. +const DATABRICKS_V2_CLAUDE_NAMES: &[&str] = &["claude", "opus", "sonnet", "haiku", "mythos", "fable"]; +/// Split a Databricks v2 endpoint name into its lowercase alphanumeric segments, +/// breaking on any non-alphanumeric delimiter (`-`, `_`, `.`, `/`, …). E.g. +/// `Databricks-Claude-Opus-5` -> `["databricks", "claude", "opus", "5"]`. +fn model_name_segments(model: &str) -> Vec { + model + .split(|c: char| !c.is_ascii_alphanumeric()) + .filter(|s| !s.is_empty()) + .map(str::to_ascii_lowercase) + .collect() +} + fn databricks_v2_route_for_model(model: &str) -> DatabricksV2Route { - // Databricks v2 catalog names identify the wire format only by these - // substrings — there is no explicit family field on the endpoint. OpenAI - // markers are checked first so a name carrying both resolves to the OpenAI - // wire (preserving the prior `gpt-5`-first behaviour). - let lower = model.to_ascii_lowercase(); - let matches_any = |markers: &[&str]| markers.iter().any(|m| lower.contains(m)); - if matches_any(DATABRICKS_V2_OPENAI_MARKERS) { + // The v2 catalog exposes no family field, so the wire format is inferred + // from the endpoint name. Discovery deliberately keeps arbitrary custom + // aliases, so we match whole name *segments* rather than raw substrings: a + // substring test would misroute unrelated names — `consolidated-llama` + // (`sol`), `terraform-coder` (`terra`), `corpus-reranker`/`octopus-model` + // (`opus`) — onto a wire whose request shape their backend can't parse, + // turning a caching optimization into a hard request/parse failure. Segment + // matching still accepts real prefixed names like `goose-opus-5`. + let segments = model_name_segments(model); + let has_named_segment = + |names: &[&str]| segments.iter().any(|seg| names.contains(&seg.as_str())); + // `gpt` family: any segment beginning with `gpt` — covers `gpt`, `gpt5`, and + // the `gpt` segment of a split `gpt-5`, without matching mid-word. + let is_gpt_family = segments.iter().any(|seg| seg.starts_with("gpt")); + // OpenAI is checked before Claude so a name carrying both markers resolves + // to the OpenAI wire (preserving the prior `gpt-5`-first precedence). + if is_gpt_family || has_named_segment(DATABRICKS_V2_OPENAI_CODE_NAMES) { DatabricksV2Route::OpenAiResponses - } else if matches_any(DATABRICKS_V2_CLAUDE_MARKERS) { + } else if has_named_segment(DATABRICKS_V2_CLAUDE_NAMES) { DatabricksV2Route::AnthropicMessages } else { DatabricksV2Route::MlflowChatCompletions @@ -2606,6 +2628,8 @@ mod tests { "/ai-gateway/openai/v1/responses", ), ("gpt-4o", OpenAiResponses, "/ai-gateway/openai/v1/responses"), + // The intentional dashless `gpt5` spelling still routes to OpenAI. + ("gpt5", OpenAiResponses, "/ai-gateway/openai/v1/responses"), ( "databricks-gpt-5-6-luna", OpenAiResponses, @@ -2671,6 +2695,31 @@ mod tests { MlflowChatCompletions, "/ai-gateway/mlflow/v1/chat/completions", ), + // Collision guard: short code names must match only as whole + // segments, never as substrings of an unrelated custom alias. + // Each of these embeds a marker (`sol`, `terra`, `opus`) mid-word + // and must stay on the MLflow fallback, not adopt a wire its + // backend can't parse. + ( + "consolidated-llama", + MlflowChatCompletions, + "/ai-gateway/mlflow/v1/chat/completions", + ), + ( + "terraform-coder", + MlflowChatCompletions, + "/ai-gateway/mlflow/v1/chat/completions", + ), + ( + "corpus-reranker", + MlflowChatCompletions, + "/ai-gateway/mlflow/v1/chat/completions", + ), + ( + "octopus-model", + MlflowChatCompletions, + "/ai-gateway/mlflow/v1/chat/completions", + ), ] { let got = databricks_v2_route_for_model(model); assert_eq!(got, route, "model={model}");