Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 104 additions & 89 deletions Cargo.lock

Large diffs are not rendered by default.

40 changes: 39 additions & 1 deletion crates/buzz-agent/src/llm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1129,6 +1129,9 @@ fn parse_responses(v: Value) -> Result<LlmResponse, AgentError> {
Some("completed") => ProviderStop::EndTurn,
_ => ProviderStop::Other,
};
if text.is_empty() && !reasoning.is_empty() && tool_calls.is_empty() {
text = reasoning.clone();
}
let input_tokens = sum_usage(&v, &["input_tokens"]);
let output_tokens = sum_usage(&v, &["output_tokens"]);
// The Responses API nests the cache split under `input_tokens_details`.
Expand Down Expand Up @@ -1318,7 +1321,7 @@ fn parse_openai(v: Value) -> Result<LlmResponse, AgentError> {
let msg = choice
.get("message")
.ok_or_else(|| AgentError::Llm("missing message".into()))?;
let text = str_field(msg, "content");
let mut text = str_field(msg, "content");
// DeepSeek and vLLM-style OpenAI-compat hosts expose reasoning tokens on the
// message object. Prefer `reasoning_content` (DeepSeek's field name); fall
// back to `reasoning` (some other providers). Both are absent for standard
Expand Down Expand Up @@ -1347,6 +1350,9 @@ fn parse_openai(v: Value) -> Result<LlmResponse, AgentError> {
)?);
}
}
if text.is_empty() && !reasoning.is_empty() && tool_calls.is_empty() {
text = reasoning.clone();
}
let input_tokens = openai_chat_input_tokens(&v);
let output_tokens = sum_usage(&v, &["completion_tokens"]);
let cached_input_tokens = openai_chat_cached_tokens(&v);
Expand Down Expand Up @@ -2524,6 +2530,21 @@ mod tests {
assert_eq!(r.stop, ProviderStop::EndTurn);
}

#[test]
fn parse_responses_uses_reasoning_when_text_is_empty() {
let v = serde_json::json!({
"status": "completed",
"output": [{
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "local model reply"}],
}],
});
let r = parse_responses(v).unwrap();
assert_eq!(r.text, "local model reply");
assert_eq!(r.reasoning, "local model reply");
assert!(r.tool_calls.is_empty());
}

#[test]
fn parse_responses_completed_with_function_call_is_tool_use() {
let v = serde_json::json!({
Expand Down Expand Up @@ -3703,6 +3724,23 @@ mod tests {
assert_eq!(parse_openai(v).unwrap().input_tokens, Some(123));
}

#[test]
fn parse_openai_uses_reasoning_content_when_content_is_empty() {
let v = serde_json::json!({
"choices": [{
"finish_reason": "stop",
"message": {
"content": "",
"reasoning_content": "local model reply"
}
}]
});
let r = parse_openai(v).unwrap();
assert_eq!(r.text, "local model reply");
assert_eq!(r.reasoning, "local model reply");
assert!(r.tool_calls.is_empty());
}

#[test]
fn parse_openai_databricks_prompt_tokens_already_inclusive() {
// Databricks' MLflow route uses the OpenAI chat wire format
Expand Down
4 changes: 2 additions & 2 deletions crates/buzz-relay/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,8 @@ async-compression = { version = "0.4.42", features = ["tokio", "gzip"] }
dev = ["buzz-auth/dev"]

[dev-dependencies]
mesh-llm-sdk = { git = "https://github.com/Mesh-LLM/mesh-llm.git", tag = "v0.73.1", package = "mesh-llm-sdk", default-features = false, features = ["client", "serving"] }
mesh-llm-host-runtime = { git = "https://github.com/Mesh-LLM/mesh-llm.git", tag = "v0.73.1", package = "mesh-llm-host-runtime", default-features = false, features = ["dynamic-native-runtime"] }
mesh-llm-sdk = { git = "https://github.com/Mesh-LLM/mesh-llm.git", tag = "v0.74.0", package = "mesh-llm-sdk", default-features = false, features = ["client", "serving"] }
mesh-llm-host-runtime = { git = "https://github.com/Mesh-LLM/mesh-llm.git", tag = "v0.74.0", package = "mesh-llm-host-runtime", default-features = false, features = ["dynamic-native-runtime"] }
buzz-core = { workspace = true, features = ["test-utils"] }
buzz-auth = { workspace = true, features = ["dev"] }
reqwest = { workspace = true }
Expand Down
175 changes: 158 additions & 17 deletions crates/buzz-relay/examples/mesh_agent_e2e.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@
//! P2 auto-model chat — agent sends `model: "auto"`; mesh router picks.
//! P3 context-fit regression — an oversized output budget (150k tokens)
//! must FAIL with the router's context error (proves the router's fit
//! gate — the failure mode the 1024 preset cap protects against).
//! gate — the failure mode the 4096 preset cap protects against).
//! P4 agentic tool use — agent + buzz-dev-mcp writes a file on disk.
//! P5 Buzz reply command — agent asks the real developer MCP to run the
//! exact `buzz messages send` command used to publish a reply.
//!
//! The serve node is the same `mesh_llm_sdk::serve` path Share-compute uses
//! (publish off, mdns, loopback). The agent legs spawn the real
Expand All @@ -26,14 +28,43 @@ use mesh_llm_sdk::{serve, MeshDiscoveryMode};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
use tokio::process::{Child, Command};

// Qwen3-8B: cached GGUF *and* complete layer package on this class of
// machine, so the serve node starts in seconds. Qwen3-30B-A3B works too but
// mesh-llm serves it from layer packages and will download them on first
// serve (~7GB) — fine in the app (progress UI), too slow for a smoke.
const DEFAULT_MODEL: &str = "unsloth/Qwen3-8B-GGUF:Q4_K_M";
// Non-reasoning Gemma 4 is the small Buzz-curated default. It reaches tool
// calls promptly instead of spending the output budget in a hidden reasoning
// loop. MESH_E2E_MODEL can still exercise any other served model explicitly.
const DEFAULT_MODEL: &str = "unsloth/gemma-4-E4B-it-GGUF:Q4_K_M";
const API_PORT: u16 = 19437;
const CONSOLE_PORT: u16 = 13231;

#[derive(Clone)]
struct McpServerEnv {
name: String,
value: String,
}

#[derive(Clone)]
struct McpServerSpec {
name: String,
command: String,
env: Vec<McpServerEnv>,
}

impl McpServerSpec {
fn new(name: impl Into<String>, command: impl Into<String>) -> Self {
Self {
name: name.into(),
command: command.into(),
env: Vec::new(),
}
}

fn set_env(&mut self, name: impl Into<String>, value: impl Into<String>) {
self.env.push(McpServerEnv {
name: name.into(),
value: value.into(),
});
}
}

fn main() -> anyhow::Result<()> {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
Expand Down Expand Up @@ -167,7 +198,7 @@ async fn run() -> anyhow::Result<()> {
let prompt = format!(
"Use your developer tools to create {marker_name} in the current working directory containing exactly the text BUZZ_OK (no quotes, no newline commentary). Then confirm."
);
let mcp = vec![("dev".to_string(), repo_bin("buzz-dev-mcp")?)];
let mcp = vec![McpServerSpec::new("dev", repo_bin("buzz-dev-mcp")?)];
let (r, marker) =
agent_chat_with_marker(&base, &served_id, None, &prompt, &mcp, &marker_name).await;
let file_ok = std::fs::read_to_string(&marker)
Expand All @@ -187,6 +218,40 @@ async fn run() -> anyhow::Result<()> {
}
let _ = std::fs::remove_file(&marker);

// P5: prove the model follows Buzz's real reply path. buzz-dev-mcp always
// prepends its own multicall `buzz` shim to PATH, so a fake `buzz` later on
// PATH cannot isolate this test. Instead, inject a fake shell into the real
// MCP server. It records the exact command at the execution boundary and
// returns the same accepted-write shape as buzz-cli, without contacting a
// community.
let send_marker = format!("mesh-e2e-buzz-send-{}.txt", std::process::id());
let prompt = "Use the developer shell tool to run exactly this command: buzz messages send --channel test-channel --content BUZZ_REPLY_OK. Do not simulate or substitute another command. After it succeeds, confirm briefly.";
let (r, marker) =
agent_chat_with_fake_shell(&base, &served_id, prompt, &mcp, &send_marker).await;
let recorded = std::fs::read_to_string(&marker).unwrap_or_default();
let send_ok =
recorded.trim() == "buzz messages send --channel test-channel --content BUZZ_REPLY_OK";
match r {
Ok(text) => record(
"P5 Buzz reply command",
send_ok,
if send_ok {
format!("accepted send command; agent said: {text}")
} else {
format!(
"unexpected command {:?}; agent said: {text}",
recorded.trim()
)
},
),
Err(e) => record(
"P5 Buzz reply command",
send_ok,
format!("agent error: {e}; recorded command: {:?}", recorded.trim()),
),
}
let _ = std::fs::remove_file(&marker);

eprintln!("[e2e] {pass} passed, {fail} failed");
if fail > 0 {
eprintln!("[e2e] FAIL: {fail} permutation(s) failed");
Expand Down Expand Up @@ -228,10 +293,11 @@ async fn agent_chat(
model: &str,
max_output_tokens: Option<&str>,
prompt: &str,
mcp_servers: &[(String, String)],
mcp_servers: &[McpServerSpec],
) -> anyhow::Result<String> {
let (result, _) =
agent_chat_in_isolated_home(base, model, max_output_tokens, prompt, mcp_servers).await;
agent_chat_in_isolated_home(base, model, max_output_tokens, prompt, mcp_servers, None)
.await;
result
}

Expand All @@ -240,11 +306,25 @@ async fn agent_chat_with_marker(
model: &str,
max_output_tokens: Option<&str>,
prompt: &str,
mcp_servers: &[(String, String)],
mcp_servers: &[McpServerSpec],
marker_name: &str,
) -> (anyhow::Result<String>, std::path::PathBuf) {
let (result, home) =
agent_chat_in_isolated_home(base, model, max_output_tokens, prompt, mcp_servers, None)
.await;
(result, home.join(marker_name))
}

async fn agent_chat_with_fake_shell(
base: &str,
model: &str,
prompt: &str,
mcp_servers: &[McpServerSpec],
marker_name: &str,
) -> (anyhow::Result<String>, std::path::PathBuf) {
let (result, home) =
agent_chat_in_isolated_home(base, model, max_output_tokens, prompt, mcp_servers).await;
agent_chat_in_isolated_home(base, model, None, prompt, mcp_servers, Some(marker_name))
.await;
(result, home.join(marker_name))
}

Expand All @@ -253,7 +333,8 @@ async fn agent_chat_in_isolated_home(
model: &str,
max_output_tokens: Option<&str>,
prompt: &str,
mcp_servers: &[(String, String)],
mcp_servers: &[McpServerSpec],
fake_shell_marker_name: Option<&str>,
) -> (anyhow::Result<String>, std::path::PathBuf) {
let agent = match repo_bin("buzz-agent") {
Ok(agent) => agent,
Expand All @@ -265,10 +346,28 @@ async fn agent_chat_in_isolated_home(
return (Err(error.into()), home);
}

let child_path = std::env::var("PATH").unwrap_or_default();
let mut effective_mcp_servers = mcp_servers.to_vec();
if let Some(marker_name) = fake_shell_marker_name {
let fake_shell = match install_fake_shell(&home) {
Ok(fake_shell) => fake_shell,
Err(error) => return (Err(error), home),
};
let marker = home.join(marker_name);
let Some(dev_server) = effective_mcp_servers
.iter_mut()
.find(|server| server.name == "dev")
else {
return (Err(anyhow::anyhow!("P5 requires the dev MCP server")), home);
};
dev_server.set_env("BUZZ_SHELL", fake_shell.to_string_lossy());
dev_server.set_env("BUZZ_E2E_SEND_MARKER", marker.to_string_lossy());
}

let mut command = Command::new(&agent);
command
.env_clear()
.env("PATH", std::env::var("PATH").unwrap_or_default())
.env("PATH", child_path)
.env("HOME", &home)
// Exactly the environment apply_relay_mesh_env() supplies.
.env("BUZZ_AGENT_PROVIDER", "openai")
Expand All @@ -293,15 +392,47 @@ async fn agent_chat_in_isolated_home(
Err(error) => return (Err(error.into()), home),
};

let result = drive_acp(&mut child, prompt, mcp_servers, &home).await;
let result = drive_acp(&mut child, prompt, &effective_mcp_servers, &home).await;
let _ = child.kill().await;
(result, home)
}

#[cfg(unix)]
fn install_fake_shell(home: &std::path::Path) -> anyhow::Result<std::path::PathBuf> {
use std::os::unix::fs::PermissionsExt;

let bin_dir = home.join("bin");
std::fs::create_dir_all(&bin_dir)?;
let executable = bin_dir.join("mesh-e2e-shell");
std::fs::write(
&executable,
r#"#!/bin/sh
if [ "$1" = "-lc" ]; then
printf '%s\n' "$2" > "$BUZZ_E2E_SEND_MARKER"
fi
if [ "$1" = "-lc" ] && [ "$2" = "buzz messages send --channel test-channel --content BUZZ_REPLY_OK" ]; then
printf '%s\n' '{"event_id":"mesh-e2e-event","accepted":true,"message":"ok"}'
exit 0
fi
printf 'unsupported fake shell invocation: %s\n' "$*" >&2
exit 2
"#,
)?;
let mut permissions = std::fs::metadata(&executable)?.permissions();
permissions.set_mode(0o755);
std::fs::set_permissions(&executable, permissions)?;
Ok(bin_dir)
}

#[cfg(not(unix))]
fn install_fake_shell(_home: &std::path::Path) -> anyhow::Result<std::path::PathBuf> {
anyhow::bail!("the mesh agent hardware harness currently requires Unix")
}

async fn drive_acp(
child: &mut Child,
prompt: &str,
mcp_servers: &[(String, String)],
mcp_servers: &[McpServerSpec],
cwd: &std::path::Path,
) -> anyhow::Result<String> {
let mut stdin = child
Expand All @@ -316,8 +447,18 @@ async fn drive_acp(

let mcp_json: Vec<serde_json::Value> = mcp_servers
.iter()
.map(|(name, command)| {
serde_json::json!({ "name": name, "command": command, "args": [], "env": [] })
.map(|server| {
let env: Vec<serde_json::Value> = server
.env
.iter()
.map(|entry| serde_json::json!({ "name": &entry.name, "value": &entry.value }))
.collect();
serde_json::json!({
"name": &server.name,
"command": &server.command,
"args": [],
"env": env,
})
})
.collect();

Expand Down
Loading
Loading