diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index ce2d8f5d..c86f9a09 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -422,6 +422,7 @@ gw.route.failed gw.route.no_upstream_alive gw.route.protocol_mismatch gw.route.rule_failed +gw.route.sealed_upstream_gone gw.route.selected_upstream_missing gw.route.upstream_missing gw.toolcall.connection_cut @@ -439,6 +440,7 @@ gw.upstream.idle_timeout_mid_stream gw.upstream.paused gw.upstream.period_used_up gw.upstream.rate_limited +gw.upstream.sealed_elsewhere gw.upstream.sign_failed gw.upstream.status gw.upstream.status_message diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index eb236734..8b100d50 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -2075,6 +2075,9 @@ slug_enum! { Turn = "turn", /// 新的一轮,而上一次回答读写的缓存值得留、又没凉 Cache = "cache", + /// 对话里带着上次回答的那一家封存的内容(Responses 的 `encrypted_content`),别家解不开。 + /// 封存的不只是推理时,候选里也只有它 + Sealed = "sealed", } } diff --git a/crates/tw-dialect/src/lib.rs b/crates/tw-dialect/src/lib.rs index a33126e4..4f306000 100644 --- a/crates/tw-dialect/src/lib.rs +++ b/crates/tw-dialect/src/lib.rs @@ -9,7 +9,8 @@ //! `usage()`),拼上游地址([`url`]),去掉 DeepSeek Harness 只发给 DeepSeek 的 //! 扩展([`harness`]),在原文上找调用方的正文([`caller`],内容过滤读它、删它), //! 读写各格式里名字不同的请求参数([`params`]),Codex 的远程压缩转给别家时怎么做 -//! ([`compaction`]),以及请求体上的提示缓存断点([`cache`])。 +//! ([`compaction`]),请求体上的提示缓存断点([`cache`]),以及上游封存、只有那个账号读得出来 +//! 的内容([`sealed`])。 pub mod anthropic; pub mod bedrock; @@ -25,6 +26,7 @@ pub mod ir; pub mod official; pub mod params; pub mod responses; +pub mod sealed; pub mod think; pub mod url; pub mod usage; diff --git a/crates/tw-dialect/src/sealed.rs b/crates/tw-dialect/src/sealed.rs new file mode 100644 index 00000000..29242257 --- /dev/null +++ b/crates/tw-dialect/src/sealed.rs @@ -0,0 +1,174 @@ +//! 上游封存的内容:只有封存它的那个账号读得出来的密文。 +//! +//! OpenAI 的 Responses 不在服务端存对话时(`store: false`,Codex 一律这样发),把下一轮还要 +//! 用的东西加密交给客户端,客户端每一轮原样带回:推理(`reasoning` 条目的 +//! `encrypted_content`)、远程压缩(`compaction`)、Codex 多代理之间传的话(`agent_message` +//! 里 `encrypted_content` 那一段)、工具结果里加密的那段。**密钥在 OpenAI 那边、按账号分**: +//! 换了一个账号或者一家中转,对方解不开,整个请求被拒(`invalid_encrypted_content`)。客户端 +//! 每一轮都带着整段历史,于是这段对话从此每一轮都被拒。 +//! +//! 能去掉的只有推理:那是模型写给自己的笔记,去掉了对话照样接得上。别的去掉就丢了内容 —— +//! 多代理之间传的话,加密的就是正文本身。所以分两种([`Sealed`])。 +//! +//! 网关自己写的「密文」([`crate::ir::CARRIED`] 开头:转换时替别家带着的签名、压缩摘要) +//! 不算:那是网关读得懂的,换到哪一家都能接着转。 + +use serde_json::Value; + +use crate::ir::{CARRIED, Dialect}; + +/// 请求里带着上游封存的内容。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Sealed { + /// 只有封存的推理:别家解不开时去掉就能接着发 + Reasoning, + /// 还有去不掉的:压缩的前文、代理之间传的话、加密的工具结果。只有封存它的那个账号接得下去 + Content, +} + +/// Responses 请求体里带着的上游封存的内容。没带的是 None。 +/// +/// 看 `input` 里每一项的每一层:哪儿有不是网关写的 `encrypted_content`,这一项就是封存的。 +/// 不按条目类型列举 —— 新出来的一种封存(多代理的话就是后来才有的)照样认得出。 +pub fn sealed(body: &Value) -> Option { + let mut found = None; + for item in body.get("input")?.as_array()? { + if !carries(item) { + continue; + } + if item.get("type").and_then(Value::as_str) == Some("reasoning") { + found = Some(Sealed::Reasoning); + } else { + return Some(Sealed::Content); + } + } + found +} + +fn carries(v: &Value) -> bool { + match v { + Value::Object(o) => o.iter().any(|(k, v)| { + (k == "encrypted_content" + && v.as_str() + .is_some_and(|s| !s.is_empty() && !s.starts_with(CARRIED))) + || carries(v) + }), + Value::Array(a) => a.iter().any(carries), + _ => false, + } +} + +/// 上游这个错误是在拒绝别的账号封存的内容吗。`wire` 是发给它的那一份的格式,`said` 是 +/// 错误正文或流里那一帧错误。 +/// +/// 看原话,不看错误外壳的结构:同一句话,OpenAI 放在 `error.code`,Codex 后端和中转各有 +/// 各的包法,有的只有一句话。 +/// +/// - OpenAI:`invalid_encrypted_content`(「The encrypted content for item rs_… could not be +/// verified」「Encrypted function output content could not be decrypted or decoded」); +/// - Anthropic:「Invalid `signature` in `thinking` block」。 +pub fn refused(wire: Dialect, said: &[u8]) -> bool { + let text = String::from_utf8_lossy(said).to_ascii_lowercase(); + match wire { + Dialect::Responses => { + text.contains("invalid_encrypted_content") + || (text.contains("encrypted") + && [ + "could not be verified", + "could not be decrypted", + "could not decrypt", + ] + .iter() + .any(|w| text.contains(w))) + } + Dialect::Anthropic => { + text.contains("invalid") && text.contains("signature") && text.contains("thinking") + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn input(items: Value) -> Value { + json!({"model": "gpt-5", "input": items}) + } + + #[test] + fn sealed_reasoning_alone_can_be_left_out() { + let body = input(json!([ + {"role": "user", "content": "hi"}, + {"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "gAAAAB"}, + {"type": "function_call", "call_id": "c1", "name": "ls", "arguments": "{}"}, + ])); + assert_eq!(sealed(&body), Some(Sealed::Reasoning)); + } + + #[test] + fn what_the_agents_said_to_each_other_cannot() { + // Codex 多代理:子代理交回来的话整段加密,发信人和任务名是明文 + let body = input(json!([ + {"type": "reasoning", "encrypted_content": "gAAAAB"}, + {"type": "agent_message", "author": "/root/worker", "recipient": "/root", "content": [ + {"type": "input_text", "text": "Message Type: MESSAGE\nPayload:\n"}, + {"type": "encrypted_content", "encrypted_content": "gAAAAC"} + ]}, + ])); + assert_eq!(sealed(&body), Some(Sealed::Content)); + // 远程压缩的前文、加密的工具结果也是 + for item in [ + json!({"type": "compaction", "encrypted_content": "gAAAAD"}), + json!({"type": "function_call_output", "call_id": "c1", "output": [ + {"type": "encrypted_content", "encrypted_content": "gAAAAE"} + ]}), + ] { + assert_eq!(sealed(&input(json!([item]))), Some(Sealed::Content)); + } + } + + #[test] + fn what_the_gateway_wrote_itself_and_empty_fields_are_not_sealed() { + let body = input(json!([ + {"type": "reasoning", "encrypted_content": "tw1.a.sig"}, + {"type": "reasoning", "encrypted_content": ""}, + {"type": "compaction", "encrypted_content": "tw1.c.aGk_"}, + {"role": "user", "content": "hi"}, + ])); + assert_eq!(sealed(&body), None); + assert_eq!(sealed(&json!({"messages": []})), None); + } + + #[test] + fn refusals_to_read_sealed_content_are_recognised() { + for said in [ + json!({"error": {"message": "The encrypted content for item rs_68 could not be verified.", + "type": "invalid_request_error", "code": "invalid_encrypted_content"}}) + .to_string(), + // Codex 后端在流里报的,只有这一句 + "Encrypted function output content could not be decrypted or decoded.".into(), + r#"{"detail":"Could not decrypt the provided encrypted_content."}"#.into(), + ] { + assert!(refused(Dialect::Responses, said.as_bytes()), "{said}"); + } + let other = json!({"error": {"message": "max_output_tokens is too large", "code": "invalid_value"}}); + assert!(!refused(Dialect::Responses, other.to_string().as_bytes())); + let anthropic = json!({"type": "error", "error": {"type": "invalid_request_error", + "message": "messages.1.content.0: Invalid `signature` in `thinking` block"}}); + assert!(refused( + Dialect::Anthropic, + anthropic.to_string().as_bytes() + )); + let budget = json!({"type": "error", "error": {"type": "invalid_request_error", + "message": "max_tokens: must be greater than thinking.budget_tokens"}}); + assert!(!refused(Dialect::Anthropic, budget.to_string().as_bytes())); + // 格式对不上:Anthropic 的那句不算 Responses 的,别的格式没有封存 + assert!(!refused( + Dialect::Responses, + anthropic.to_string().as_bytes() + )); + assert!(!refused(Dialect::Chat, other.to_string().as_bytes())); + } +} diff --git a/crates/tw-gateway/src/affinity.rs b/crates/tw-gateway/src/affinity.rs index 57de1abd..b027b2dd 100644 --- a/crates/tw-gateway/src/affinity.rs +++ b/crates/tw-gateway/src/affinity.rs @@ -20,8 +20,14 @@ //! 对话按「路由 + 对话标识」记(见 [`Conversation`]),回答的那一家还要对得上策略组: //! 同一段对话这一轮被规则送去另一个组,上次那一家的缓存和这一组无关。 //! -//! **只在内存里**,最多记 [`CAP`] 段、每段最多 [`KEEP_MS`]:core 重启之后第一轮 -//! 按策略重新排,最坏是一次缓存重建。**跨配置重载存活**,但沿用的路由决定只在同一 +//! **带着上游封存内容的对话另当别论**(见 [`tw_dialect::sealed`]):那段历史只有封存它的 +//! 那个账号读得出来,换到别家整个请求被拒,而且之后每一轮都被拒。所以每段对话还记着上次 +//! 回答它的那一家([`Affinity::holder`]),不管策略组、不管缓存凉没凉、配置改没改、那一家 +//! 中途没没了声音。怎么用它见 `server::pipeline::route`。 +//! +//! **只在内存里**,最多记 [`CAP`] 段、每段最多 [`KEEP_MS`](带过封存内容的 [`SEALED_KEEP_MS`]): +//! core 重启之后第一轮按策略重新排,最坏是一次缓存重建 —— 带着封存内容的对话这时认不出 +//! 是谁封存的,可能被送去解不开的那一家。**跨配置重载存活**,但沿用的路由决定只在同一 //! 份配置里有效(配置改了,规则可能已经不是那条了)。 use std::collections::HashMap; @@ -44,6 +50,10 @@ pub const CACHE_COLD_MS: u64 = 5 * 60 * 1000; /// 一段对话记多久。一天没动静的对话,缓存早就没了,沿用的路由决定也不再有意义 pub const KEEP_MS: u64 = 24 * 60 * 60 * 1000; +/// 带过封存内容的对话记多久。缓存早就凉了,可那段历史还是只有那一家读得出来:客户端 +/// 隔几天接着这段对话说话(Codex 的 `resume`)也得送回那一家 +pub const SEALED_KEEP_MS: u64 = 30 * KEEP_MS; + /// 最多记几段对话。满了先清掉超期的,还满就清最久没动静的那段 pub const CAP: usize = 4096; @@ -142,6 +152,11 @@ struct Entry { held: Option, /// 上次回答它的那一家 answer: Option, + /// 上次回答它的那一家,**只给封存的内容用**(见 [`Affinity::holder`]):和 `answer` 不同, + /// 那一家没了声音也不放开 —— 换到别家也解不开 + holder: Option, + /// 这段对话带过封存的内容:记得久一些([`SEALED_KEEP_MS`]) + sealed: bool, } struct Held { @@ -180,7 +195,7 @@ impl Affinity { return None; } let map = self.lock(); - let e = map.get(&c.key).filter(|e| fresh(e, now_ms))?; + let e = map.get(&c.key).filter(|e| recent(e, now_ms))?; let h = e.held.as_ref()?; (h.turn == c.turn.number && Weak::ptr_eq(&h.engine, &Arc::downgrade(engine))) .then(|| h.decision.clone()) @@ -216,7 +231,7 @@ impl Affinity { let map = self.lock(); let a = map .get(&c.key) - .filter(|e| fresh(e, now_ms))? + .filter(|e| recent(e, now_ms))? .answer .as_ref()?; if a.group.as_deref() != group { @@ -262,7 +277,22 @@ impl Affinity { at_ms: now_ms, cache, }; - self.upsert(&t.key, now_ms, |e| e.answer = Some(answer)); + self.upsert(&t.key, now_ms, |e| { + e.holder = Some(answer.provider.clone()); + e.answer = Some(answer); + }); + } + + /// 这段对话带着上游封存的内容:上次回答它的那一家 —— 请求里最新的那几段是它封存的,前面 + /// 的它也读得出来(不然它回答不了)。认不出是谁的(没回答过、core 重启过、记录超期了)是 + /// None。 + /// + /// 记下这段对话带过封存的内容:之后记得久一些([`SEALED_KEEP_MS`]) + pub fn holder(&self, c: &Conversation, now_ms: u64) -> Option { + let mut map = self.lock(); + let e = map.get_mut(&c.key).filter(|e| fresh(e, now_ms))?; + e.sealed = true; + e.holder.clone() } /// `provider` 在这段对话上没了声音(无响应超时,见 `server::pipeline::idle`):上次回答它 @@ -298,12 +328,19 @@ impl Affinity { touched_ms: now_ms, held: None, answer: None, + holder: None, + sealed: false, }); - // 超期的整段作废:一天前那一轮的决定和回答,对现在的请求都没有意义 - if !fresh(e, now_ms) { + // 超期的作废:一天前那一轮的决定和回答,对现在的请求都没有意义。封存的内容是谁的, + // 记到它自己的期限 + if !recent(e, now_ms) { e.held = None; e.answer = None; } + if !fresh(e, now_ms) { + e.holder = None; + e.sealed = false; + } e.touched_ms = e.touched_ms.max(now_ms); f(e); } @@ -314,10 +351,17 @@ impl Affinity { } } -fn fresh(e: &Entry, now_ms: u64) -> bool { +/// 这一轮的决定、上次的回答还算数:一天之内动过 +fn recent(e: &Entry, now_ms: u64) -> bool { now_ms.saturating_sub(e.touched_ms) <= KEEP_MS } +/// 这段对话还记着:带过封存内容的记 [`SEALED_KEEP_MS`],别的和 [`recent`] 一样 +fn fresh(e: &Entry, now_ms: u64) -> bool { + let keep = if e.sealed { SEALED_KEEP_MS } else { KEEP_MS }; + now_ms.saturating_sub(e.touched_ms) <= keep +} + /// 这一次由谁回答。**跟着请求的结局走**:上游回了成功、响应走完了才记下它和它读写 /// 了多少缓存(见 [`crate::ending::Ending`])。 pub struct Ticket { @@ -599,6 +643,33 @@ mod tests { assert_eq!(a.stay(&conv(1, true), Some("池子"), &mut c, up, MIN), None); } + #[test] + fn the_holder_of_sealed_content_outlasts_what_the_cache_would_keep() { + let a = Arc::new(Affinity::default()); + let c = conv(1, false); + assert_eq!(a.holder(&c, 0), None, "没回答过:认不出是谁的"); + answer(&a, &c, "乙", 0, 0); + // 没了声音也照记:换到别家也解不开 + a.left(&c, "乙"); + assert_eq!(a.holder(&c, MIN).as_deref(), Some("乙")); + // 带过封存内容的对话记一个月:隔几天接着说话也送得回去,缓存那一份早就作废了 + let later = 3 * KEEP_MS; + assert_eq!(a.holder(&c, later).as_deref(), Some("乙")); + let mut cands = pool(); + assert_eq!( + a.stay(&conv(1, true), Some("池子"), &mut cands, up, later), + None + ); + // 之后回答它的是丙:从此是丙的 + answer(&a, &c, "丙", 0, later); + assert_eq!(a.holder(&c, later + MIN).as_deref(), Some("丙")); + assert_eq!(a.holder(&c, later + SEALED_KEEP_MS + 1), None); + // 没带过封存内容的对话照旧只记一天 + let plain = Conversation::new("default", "别的", Turn::default()); + answer(&a, &plain, "甲", 0, 0); + assert_eq!(a.holder(&plain, KEEP_MS + 1), None); + } + #[test] fn a_day_old_conversation_is_forgotten_and_the_table_stays_bounded() { let a = Arc::new(Affinity::default()); diff --git a/crates/tw-gateway/src/client_api.rs b/crates/tw-gateway/src/client_api.rs index a5d25ed6..2c3280d4 100644 --- a/crates/tw-gateway/src/client_api.rs +++ b/crates/tw-gateway/src/client_api.rs @@ -247,6 +247,9 @@ pub struct Reading { /// 请求体解得开(是 JSON)。之后每一跳按字节改它的地方(见 [`crate::egress`])凭这个 /// 知道不用再解一遍 pub json: bool, + /// 请求里带着上游封存的内容(见 [`tw_dialect::sealed`]):这段对话只有封存它的那一家 + /// 读得出来。只有生成回答的 Responses 请求会带 + pub sealed: Option, } pub fn read(path: &str, query: Option<&str>, body: Option<&serde_json::Value>) -> Reading { @@ -276,6 +279,9 @@ pub fn read(path: &str, query: Option<&str>, body: Option<&serde_json::Value>) - }, }; facts.dialect = api.map(|a| a.slug()).unwrap_or_default().to_string(); + let sealed = body + .filter(|_| generates && api == Some(ClientApi::OpenaiResponses)) + .and_then(tw_dialect::sealed::sealed); Reading { api, generates, @@ -283,6 +289,7 @@ pub fn read(path: &str, query: Option<&str>, body: Option<&serde_json::Value>) - facts, harness: None, json: body.is_some(), + sealed, } } diff --git a/crates/tw-gateway/src/ending.rs b/crates/tw-gateway/src/ending.rs index 6ebb1840..7f68a57b 100644 --- a/crates/tw-gateway/src/ending.rs +++ b/crates/tw-gateway/src/ending.rs @@ -78,6 +78,9 @@ pub struct Ending { upstream_error: Option<(String, String)>, /// 上游报错的那一帧的 `data`:回程照它认是不是限流(见 [`Ending::upstream_fault`]) upstream_fault: Option, + /// 上游报的错是解不开对话里别家封存的内容(见 [`tw_dialect::sealed`]):失败的说明讲清楚 + /// 怎么回事,不只转述原话 + sealed_refused: bool, /// 上游回的不是 2xx、原样交给了客户端(见 [`Ending::refused`])。**有它就不是成功**: /// 客户端拿到的是上游的错误,不是回答 refusal: Option, @@ -134,6 +137,9 @@ impl Refusal { }); let upstream = self.provider; match said { + Some(message) if sealed_refusal(self.dialect, &message) => { + sealed_elsewhere(upstream, message) + } Some(message) => msg!( "gw.upstream.status_message", upstream = upstream, status = status, message = message => @@ -147,6 +153,23 @@ impl Refusal { } } +/// 上游的错误是解不开对话里别家封存的内容吗(见 [`tw_dialect::sealed::refused`])。只认 +/// Responses 的:Anthropic 拒绝别家签的推理时,网关去掉推理再发一次就接得下去(见 +/// [`crate::seal`]),不用让人换上游 +fn sealed_refusal(dialect: ir::Dialect, said: &str) -> bool { + dialect == ir::Dialect::Responses && tw_dialect::sealed::refused(dialect, said.as_bytes()) +} + +/// 上游解不开对话里别家封存的内容。**说清楚怎么办**:原话只说解不开,看不出是换了上游 +fn sealed_elsewhere(upstream: String, message: String) -> Msg { + msg!( + "gw.upstream.sealed_elsewhere", upstream = upstream, message = message => + "Upstream `{upstream}` cannot read this conversation's encrypted history: another \ + upstream encrypted it, and only that one can read it. Route the conversation back to \ + that upstream, or start a new conversation. The upstream said: {message}" + ) +} + /// 上游在错误正文里说的那句话:按它的格式读出来的说明,读不出来的就是正文本身。**网页 /// 不算**(代理、防火墙回的那种错误页):一页 HTML 的开头说明不了什么。最多 [`SAID_MAX`] /// 个字。 @@ -321,6 +344,7 @@ impl Ending { watch: None, upstream_error: None, upstream_fault: None, + sealed_refused: false, refusal: None, answer: None, abort: None, @@ -499,6 +523,7 @@ impl Ending { }; let frames = w.frames.feed(chunk); if let Some((found, data)) = w.error_in(frames) { + self.sealed_refused = sealed_refusal(w.dialect, &data); self.upstream_error = Some(found); self.upstream_fault = Some(data); self.watch = None; @@ -615,19 +640,22 @@ impl Ending { if self.upstream_error.is_none() && let Some((found, data)) = w.error_in(frames) { + self.sealed_refused = sealed_refusal(w.dialect, &data); self.upstream_error = Some(found); self.upstream_fault = Some(data); } } if let Some((upstream, message)) = self.upstream_error.take() { let message: String = message.chars().take(500).collect(); - self.fail( - tw_api::FailureSource::Upstream, + let why = if self.sealed_refused { + sealed_elsewhere(upstream, message) + } else { msg!( "gw.upstream.stream_error", upstream = upstream, message = message => "Upstream `{upstream}` reported an error partway through the response: {message}" - ), - ); + ) + }; + self.fail(tw_api::FailureSource::Upstream, why); return; } let (usage, answered_model) = self.settle(); diff --git a/crates/tw-gateway/src/seal.rs b/crates/tw-gateway/src/seal.rs index 70fc8a28..f907f176 100644 --- a/crates/tw-gateway/src/seal.rs +++ b/crates/tw-gateway/src/seal.rs @@ -75,32 +75,10 @@ impl Refused { } } -/// 上游这个 400 是在拒绝封存的推理吗。`wire` 是发给它的那一份的格式。 -/// -/// 看原话,不看错误外壳的结构:同一句话,OpenAI 放在 `error.code`,Codex 后端和 -/// 中转各有各的包法。 +/// 上游这个 400 是在拒绝封存的推理吗。`wire` 是发给它的那一份的格式。认法见 +/// [`tw_dialect::sealed::refused`] pub fn refusal(wire: Dialect, body: &[u8]) -> bool { - if body.len() > REFUSAL_MAX { - return false; - } - let text = String::from_utf8_lossy(body).to_ascii_lowercase(); - match wire { - Dialect::Responses => { - text.contains("invalid_encrypted_content") - || ((text.contains("encrypted content") || text.contains("encrypted_content")) - && [ - "could not be verified", - "could not be decrypted", - "could not decrypt", - ] - .iter() - .any(|w| text.contains(w))) - } - Dialect::Anthropic => { - text.contains("invalid") && text.contains("signature") && text.contains("thinking") - } - _ => false, - } + body.len() <= REFUSAL_MAX && tw_dialect::sealed::refused(wire, body) } /// 去掉请求体里封存的推理。`only` 给了的只去掉其中的那些,没给的全去掉。 @@ -211,30 +189,14 @@ mod tests { } #[test] - fn openais_refusal_is_recognised_and_other_400s_are_not() { + fn a_refusal_is_read_only_from_a_short_body() { let refused = json!({"error": { "message": "The encrypted content for item rs_68 could not be verified.", - "type": "invalid_request_error", "param": null, "code": "invalid_encrypted_content"}}); - assert!(refusal(Dialect::Responses, refused.to_string().as_bytes())); - // 没带码、只有那句话的中转 - assert!(refusal( - Dialect::Responses, - br#"{"detail":"Could not decrypt the provided encrypted_content."}"# - )); - let other = json!({"error": {"message": "max_output_tokens is too large", "code": "invalid_value"}}); - assert!(!refusal(Dialect::Responses, other.to_string().as_bytes())); - // 格式不对上:Anthropic 的那句不算 Responses 的 - assert!(!refusal(Dialect::Chat, refused.to_string().as_bytes())); - } - - #[test] - fn anthropics_refusal_is_recognised() { - let refused = json!({"type": "error", "error": {"type": "invalid_request_error", - "message": "messages.1.content.0: Invalid `signature` in `thinking` block"}}); - assert!(refusal(Dialect::Anthropic, refused.to_string().as_bytes())); - let other = json!({"type": "error", "error": {"type": "invalid_request_error", - "message": "max_tokens: must be greater than thinking.budget_tokens"}}); - assert!(!refusal(Dialect::Anthropic, other.to_string().as_bytes())); + "code": "invalid_encrypted_content"}}) + .to_string(); + assert!(refusal(Dialect::Responses, refused.as_bytes())); + let huge = format!("{refused}{}", " ".repeat(REFUSAL_MAX)); + assert!(!refusal(Dialect::Responses, huge.as_bytes())); } #[test] diff --git a/crates/tw-gateway/src/server/pipeline.rs b/crates/tw-gateway/src/server/pipeline.rs index daaf3b5e..c21f633e 100644 --- a/crates/tw-gateway/src/server/pipeline.rs +++ b/crates/tw-gateway/src/server/pipeline.rs @@ -839,6 +839,30 @@ fn route( }), stayed_on: None, }; + // 带着上游封存内容的对话,只有封存它的那一家读得出来(见 `tw_dialect::sealed`)。认得出是 + // 谁封存的(上次回答它的那一家,见 `crate::affinity`),就送回它那儿:规则这一次选了谁、 + // 配置改没改、缓存凉没凉都不管。认不出来的照常路由 —— 被拒了,失败的说明会讲清楚 + let sealed = reading + .sealed + .zip(conv) + .and_then(|(kind, c)| Some((kind, state.affinity.holder(c, now)?))); + if let Some((tw_dialect::sealed::Sealed::Content, holder)) = &sealed { + // 去不掉的封存:别家一个都接不下去,候选只剩它,也不故障转移 + if !rt.config.providers.iter().any(|p| p.name == *holder) { + return Routed::Refused( + choice, + GatewayError::config(msg!( + "gw.route.sealed_upstream_gone", upstream = holder.clone() => + "This conversation's history is encrypted by upstream `{upstream}`, which is \ + no longer configured, and no other upstream can read it. Start a new \ + conversation." + )), + ); + } + decision.candidates = vec![holder.clone()]; + decision.via_group = None; + choice.group = None; + } // 每个候选实际要的模型和它的来历:规则改写过的按改写后的算;客户端写的、阶段一改写的 // 是客户端那一侧的名称(可能是别名),指定的、阶段二改的原样发出。准入看它 let asked = rt.engine.asked( @@ -878,7 +902,17 @@ fn route( conv, now, ); - if let Some(why) = arranged.stayed { + let mut stayed = arranged.stayed; + // 封存的只有推理:封存它的那一家排到头上,推理接得上;别家照旧留着做故障转移 —— 去掉推理 + // 也接得下去(见 `crate::seal`) + if let Some((_, holder)) = &sealed + && let Some(at) = decision.candidates.iter().position(|p| p == holder) + { + let p = decision.candidates.remove(at); + decision.candidates.insert(0, p); + stayed = Some(tw_api::Stay::Sealed); + } + if let Some(why) = stayed { choice.affinity = Some(tw_api::AffinityView { held_route, stayed: Some(why), diff --git a/crates/tw-gateway/tests/sealed.rs b/crates/tw-gateway/tests/sealed.rs new file mode 100644 index 00000000..9c0bc194 --- /dev/null +++ b/crates/tw-gateway/tests/sealed.rs @@ -0,0 +1,294 @@ +//! 带着上游封存内容的对话(见 `tw_dialect::sealed`)送回封存它的那一家;别家解不开时,失败的 +//! 说明讲清楚怎么回事 —— 走真实的管线,上游是两个各自封存、各自只认自己封存的假上游。 + +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Duration; + +use axum::Router; +use axum::routing::any; +use tokio::sync::broadcast::Receiver; +use tw_api::{AttemptOutcome, AttemptView, Event, Stay}; +use tw_config::{Client, Config, Provider}; +use tw_engine::{RouteSet, Rule}; + +/// 一个假的 OpenAI 账号:回答里带一段它封存的推理(`gAAAA-<名字>`),请求里有别家封存的就 +/// 拒绝 —— 和 OpenAI 一样回 400 `invalid_encrypted_content`;`in_stream` 的像 Codex 后端那样 +/// 回 200,在流里报这个错。`down` 打开时回 503。 +struct Account { + addr: SocketAddr, + hits: Arc, + down: Arc, +} + +async fn account(name: &'static str, in_stream: bool) -> Account { + let hits = Arc::new(AtomicUsize::new(0)); + let down = Arc::new(AtomicBool::new(false)); + let (h, d) = (hits.clone(), down.clone()); + let app = Router::new().fallback(any(move |body: String| { + let (h, d) = (h.clone(), d.clone()); + async move { + h.fetch_add(1, Ordering::SeqCst); + let builder = axum::response::Response::builder(); + if d.load(Ordering::SeqCst) { + return builder.status(503).body(axum::body::Body::from("down")).unwrap(); + } + let foreign = body + .match_indices("gAAAA-") + .any(|(i, _)| !body[i + 6..].starts_with(name)); + if foreign && in_stream { + let frame = r#"{"type":"error","error":{"type":"invalid_request_error","code":"invalid_encrypted_content","message":"Encrypted function output content could not be decrypted or decoded."}}"#; + return builder + .header("content-type", "text/event-stream") + .body(axum::body::Body::from(format!( + "event: error\ndata: {frame}\n\n" + ))) + .unwrap(); + } + if foreign { + return builder + .status(400) + .header("content-type", "application/json") + .body(axum::body::Body::from( + r#"{"error":{"message":"The encrypted content for item rs_1 could not be verified.","type":"invalid_request_error","param":null,"code":"invalid_encrypted_content"}}"#, + )) + .unwrap(); + } + let answer = format!( + r#"{{"id":"resp_1","object":"response","status":"completed","model":"gpt-5","output":[{{"type":"reasoning","id":"rs_1","summary":[],"encrypted_content":"gAAAA-{name}"}},{{"type":"message","role":"assistant","content":[{{"type":"output_text","text":"ok"}}]}}],"usage":{{"input_tokens":3,"output_tokens":1,"total_tokens":4}}}}"# + ); + builder + .header("content-type", "application/json") + .body(axum::body::Body::from(answer)) + .unwrap() + } + })); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + Account { addr, hits, down } +} + +fn provider(name: &str, up: &Account) -> Provider { + Provider { + name: name.into(), + base_url: format!("http://{}/v1", up.addr), + key: Some("sk-x".into()), + protocol: Some(tw_config::Protocol::OpenaiResponses), + ..Default::default() + } +} + +/// 甲乙两家。`to` 是唯一一条规则送去的地方:一家,或者轮流(`池`,没有缓存可留,每一轮 +/// 都轮到下一家) +fn cfg(accounts: &[(&str, &Account)], to: &str) -> Config { + Config { + clients: vec![Client { + name: "我".into(), + key: "tw-k".into(), + ..Default::default() + }], + providers: accounts.iter().map(|(n, a)| provider(n, a)).collect(), + groups: vec![tw_engine::Group { + name: "池".into(), + kind: tw_engine::GroupType::LoadBalance, + providers: accounts.iter().map(|(n, _)| (*n).into()).collect(), + selected: None, + balance_by: Default::default(), + }], + routes: vec![RouteSet::default_with(vec![Rule { + name: "Codex".into(), + when: serde_yaml_ng::from_str("{}").unwrap(), + to: Some(to.into()), + set: None, + deny: None, + }])], + ..Default::default() + } +} + +async fn serve(state: tw_gateway::AppState) -> (SocketAddr, Receiver) { + let events = state.bus.subscribe(); + let addr = tw_gateway::serve(state, ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + (addr, events) +} + +/// 这段对话的第 `turn` 轮:前面每一轮都由用户说一句、模型带回 `sealed` 里那家封存的推理; +/// `agents` 时最后还有一条子代理交回来的、加密了的话 +fn body(turn: usize, sealed: &str, agents: bool) -> String { + let mut input = Vec::new(); + for i in 0..turn { + input.push(format!(r#"{{"role":"user","content":"第 {i} 轮"}}"#)); + if i + 1 < turn { + input.push(format!( + r#"{{"type":"reasoning","id":"rs_{i}","summary":[],"encrypted_content":"gAAAA-{sealed}"}}"# + )); + } + } + if agents { + input.push(format!( + r#"{{"type":"agent_message","author":"/root/worker","recipient":"/root","content":[{{"type":"input_text","text":"Message Type: MESSAGE\nPayload:\n"}},{{"type":"encrypted_content","encrypted_content":"gAAAA-{sealed}"}}]}}"# + )); + } + format!( + r#"{{"model":"gpt-5","instructions":"You are Codex.","prompt_cache_key":"对话-1","store":false,"input":[{}]}}"#, + input.join(",") + ) +} + +/// 结局:路由事件里试过的上游、留下的理由,和失败时的那句话 +struct Outcome { + status: u16, + attempts: Vec, + stayed: Option, + failed: Option, +} + +async fn ask(gw: SocketAddr, rx: &mut Receiver, body: String) -> Outcome { + ask_in(gw, rx, "codex-1", body).await +} + +/// [`ask`],在会话 `session` 里 +async fn ask_in(gw: SocketAddr, rx: &mut Receiver, session: &str, body: String) -> Outcome { + let status = reqwest::Client::builder() + .no_proxy() + .build() + .unwrap() + .post(format!("http://{gw}/v1/responses")) + .header("authorization", "Bearer tw-k") + .header("session_id", session) + .body(body) + .send() + .await + .unwrap() + .status() + .as_u16(); + let mut routed = (Vec::new(), None); + loop { + let ev = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .expect("5 秒内没等到结局") + .expect("事件流断了"); + match ev { + Event::RequestRouted { + attempts, affinity, .. + } => routed = (attempts, affinity.and_then(|a| a.stayed)), + Event::RequestFinished { .. } => { + return Outcome { + status, + attempts: routed.0, + stayed: routed.1, + failed: None, + }; + } + Event::RequestFailed { message, .. } => { + return Outcome { + status, + attempts: routed.0, + stayed: routed.1, + failed: Some(message), + }; + } + _ => {} + } + } +} + +fn served_by(o: &Outcome) -> Vec<(&str, AttemptOutcome)> { + o.attempts + .iter() + .map(|a| (a.provider.as_str(), a.outcome)) + .collect() +} + +#[tokio::test] +async fn sealed_agent_messages_go_back_to_the_account_that_sealed_them() { + let (a, b) = (account("甲", false).await, account("乙", false).await); + let state = tw_gateway::AppState::new(cfg(&[("甲", &a), ("乙", &b)], "甲")).unwrap(); + let (gw, mut rx) = serve(state.clone()).await; + let o = ask(gw, &mut rx, body(1, "甲", false)).await; + assert_eq!(served_by(&o), [("甲", AttemptOutcome::Served)]); + + // 改了配置:规则现在送去乙。带着甲封存的、子代理交回来的话的这段对话照旧回甲,乙不做备选 + state.reload(cfg(&[("甲", &a), ("乙", &b)], "乙")).unwrap(); + let o = ask(gw, &mut rx, body(2, "甲", true)).await; + assert_eq!(o.status, 200); + assert_eq!(served_by(&o), [("甲", AttemptOutcome::Served)]); + assert_eq!(o.stayed, Some(Stay::Sealed)); + assert_eq!(b.hits.load(Ordering::SeqCst), 0); + + // 甲发不出去:报甲的错,不转给解不开的乙 + a.down.store(true, Ordering::SeqCst); + let o = ask(gw, &mut rx, body(3, "甲", true)).await; + assert_ne!(o.status, 200); + assert_eq!(o.attempts.len(), 1, "{:#?}", o.attempts); + assert_eq!(o.attempts[0].provider, "甲"); + assert_eq!(b.hits.load(Ordering::SeqCst), 0); + + // 新的对话照新的规则走 + let fresh = body(1, "甲", false).replace("对话-1", "对话-2"); + let o = ask_in(gw, &mut rx, "codex-2", fresh).await; + assert_eq!(served_by(&o), [("乙", AttemptOutcome::Served)]); +} + +#[tokio::test] +async fn a_conversation_whose_sealing_upstream_was_removed_is_refused_plainly() { + let (a, b) = (account("甲", false).await, account("乙", false).await); + let state = tw_gateway::AppState::new(cfg(&[("甲", &a), ("乙", &b)], "甲")).unwrap(); + let (gw, mut rx) = serve(state.clone()).await; + ask(gw, &mut rx, body(1, "甲", false)).await; + + state.reload(cfg(&[("乙", &b)], "乙")).unwrap(); + let o = ask(gw, &mut rx, body(2, "甲", true)).await; + assert_ne!(o.status, 200); + assert!(o.attempts.is_empty(), "{:#?}", o.attempts); + let m = o.failed.expect("该失败"); + assert_eq!(m.code, "gw.route.sealed_upstream_gone", "{m:?}"); + assert_eq!(b.hits.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn sealed_reasoning_alone_prefers_its_account_and_can_still_fail_over() { + let (a, b) = (account("甲", false).await, account("乙", false).await); + let state = tw_gateway::AppState::new(cfg(&[("甲", &a), ("乙", &b)], "池")).unwrap(); + let (gw, mut rx) = serve(state).await; + let first = ask(gw, &mut rx, body(1, "甲", false)).await; + let holder = first.attempts.last().unwrap().provider.clone(); + let other = if holder == "甲" { "乙" } else { "甲" }; + + // 没有缓存可留,轮流本该轮到另一家;带着封存的推理,回封存它的那一家 + for turn in 2..4 { + let o = ask(gw, &mut rx, body(turn, &holder, false)).await; + assert_eq!(served_by(&o), [(holder.as_str(), AttemptOutcome::Served)]); + assert_eq!(o.stayed, Some(Stay::Sealed)); + } + + // 那一家发不出去:转给另一家,去掉解不开的推理接着回答(见 `crate::seal`) + let (down, _) = if holder == "甲" { (&a, &b) } else { (&b, &a) }; + down.down.store(true, Ordering::SeqCst); + let o = ask(gw, &mut rx, body(4, &holder, false)).await; + assert_eq!(o.status, 200, "{:?}", o.failed); + assert_eq!(o.attempts.last().unwrap().provider, other); +} + +#[tokio::test] +async fn an_upstream_that_cannot_read_the_history_says_so_plainly() { + // 认不出是谁封存的(core 重启过、记录超期了):照规则送去乙,乙解不开 + for in_stream in [false, true] { + let b = account("乙", in_stream).await; + let state = tw_gateway::AppState::new(cfg(&[("乙", &b)], "乙")).unwrap(); + let (gw, mut rx) = serve(state).await; + let mut sent = body(2, "甲", true); + if in_stream { + sent = sent.replace(r#""store":false"#, r#""store":false,"stream":true"#); + } + let o = ask(gw, &mut rx, sent).await; + let m = o.failed.expect("该失败"); + assert_eq!(m.code, "gw.upstream.sealed_elsewhere", "{in_stream}: {m:?}"); + assert_eq!(m.args.get("upstream").map(String::as_str), Some("乙")); + } +} diff --git a/docs/config.md b/docs/config.md index 31f55de8..95371c77 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1320,6 +1320,21 @@ it to. A `load-balance` weight is therefore the long-run share of requests: conversations in progress stay where they are and count toward that upstream's share. +A conversation carrying content an upstream encrypted is a different case. +OpenAI Responses hands clients encrypted state to send back each turn +(`encrypted_content`: reasoning, compacted history, what Codex sub-agents +report to each other), and only the account that encrypted it can read it. +Such a conversation goes back to the upstream that last answered it whatever +the rule now picks, even after the configuration changed, the cache went cold +or a day has passed (it is remembered for 30 days). When it carries more than +encrypted reasoning, that upstream is the only candidate: there is no +failover, and if the upstream has been removed the request is refused with an +explanation. Encrypted reasoning alone can be left out, so other upstreams +remain as failover. After twcore restarts the gateway no longer knows which +upstream encrypted a conversation; if another upstream receives it and cannot +read it, the failure says so, and the conversation has to go back to the +original upstream or start over. + `balance_by` lets a `load-balance` group also look at how each upstream has been doing lately. Each member's weight is multiplied by a factor, and the group shares out requests by the result in the same way as above. diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 529f4b55..6579a094 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -972,6 +972,8 @@ groups: 无论哪种类型,一段对话都留在上次回答它的那一家上游,让上游缓存着的那部分被再次读取,而不是换一家全价重算。同一轮之内(客户端正在回传工具结果)一律不换;跨轮时,上一次回答读或写了至少 1024 个 token 的 prompt cache、且距今不到五分钟,才继续留下。上游因失败进入冷却时,对话随之放开;故障转移之后接下回答的那一家,就是之后留下的那一家。一轮开始时命中的规则也沿用到这一轮结束:按输入大小或图片分流的规则不会让一轮半路换家,除非输入已经超出规则所指模型的上下文窗口。因此 `load-balance` 的权重是长期看各家分到的请求的比例:进行中的对话留在原来的上游,也算进那一家的份额。 +带着上游加密内容的对话另当别论。OpenAI Responses 把下一轮还要用的内容加密交给客户端,每一轮原样带回(`encrypted_content`:推理、压缩后的前文、Codex 子代理之间传递的消息),只有加密它的那个账号读得出来。这样的对话一律送回上次回答它的那一家,不论规则现在选中哪一家,配置改过、缓存已凉或隔了一天也一样(记 30 天)。加密的不只是推理时,候选只有那一家:不做故障转移,那一家已从配置中删除时请求被拒绝并说明原因。只加密了推理时,推理可以去掉,其余上游仍作为故障转移的备选。twcore 重启之后不再知道对话由哪一家加密;送到别家而解不开时,失败说明会写明原因,这段对话需要改回原来的上游,或者重新开始。 + `balance_by` 让 `load-balance` 组再看各上游最近的表现:每个成员的权重乘上一个系数,组内请求按乘出来的结果照上文的方式分。 - `weights`(默认):只按权重。