From 6d4377afd7f026a5ad1240dcf3a53151ba870eff Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 9 Sep 2026 22:04:38 -0700 Subject: [PATCH] feat(smith): trust provider input token usage --- crates/adapter-smith/src/agent.rs | 53 ++- crates/adapter-smith/src/compact.rs | 64 ++- crates/adapter-smith/src/context.rs | 167 +++++++- crates/adapter-smith/src/interactive.rs | 45 ++- crates/adapter-smith/src/model_limits.rs | 30 +- .../adapter-smith/src/provider/anthropic.rs | 372 +++++++++++++++++- .../src/provider/antigravity_oauth.rs | 2 +- .../src/provider/claude_oauth.rs | 4 + .../adapter-smith/src/provider/codex_oauth.rs | 14 +- crates/adapter-smith/src/provider/config.rs | 31 ++ crates/adapter-smith/src/provider/gemini.rs | 2 +- crates/adapter-smith/src/provider/meta.rs | 6 +- crates/adapter-smith/src/provider/mod.rs | 15 +- crates/adapter-smith/src/provider/ollama.rs | 2 +- crates/adapter-smith/src/provider/openai.rs | 7 +- crates/daemon/src/config.rs | 8 + docs/smith.md | 45 ++- ...mith-model-profiles-are-named-endpoints.md | 17 + ...provider-token-counts-are-authoritative.md | 46 +++ ...c-prompt-cache-and-context-capabilities.md | 48 +++ 20 files changed, 873 insertions(+), 105 deletions(-) create mode 100644 specs/0215-smith-provider-token-counts-are-authoritative.md create mode 100644 specs/0216-anthropic-prompt-cache-and-context-capabilities.md diff --git a/crates/adapter-smith/src/agent.rs b/crates/adapter-smith/src/agent.rs index c6e054a5..72f3394e 100644 --- a/crates/adapter-smith/src/agent.rs +++ b/crates/adapter-smith/src/agent.rs @@ -616,6 +616,7 @@ pub async fn run( let provider = spec.provider; let mut provider_context_window = None; let mut provider_context_checked = false; + let mut input_tokens = context::InputTokenCounter::default(); // Per-model learned token limits — adapts on overflow errors // and bumps upward on successful probe calls. Shared across // all construct sessions on this machine via state_dir. @@ -864,11 +865,11 @@ pub async fn run( } let hardcoded_cap = context::context_window_tokens(provider_name, &model); let learned = limits.get(provider_name, &model); - let est = - fixed_context_tokens.saturating_add(context::estimate_tokens(&messages) as u64); + let preflight_tokens = + context::preflight_input_tokens(&input_tokens, fixed_context_tokens, &messages); let is_probe = provider_context_window.is_none() && learned.is_some() - && limits.should_probe(provider_name, &model, est, now_ms); + && limits.should_probe(provider_name, &model, preflight_tokens, now_ms); let effective_cap = provider_context_window .or(learned) .unwrap_or(hardcoded_cap as u64); @@ -877,7 +878,7 @@ pub async fn run( } else { context::UTILIZATION }; - let budget = context::message_budget(effective_cap, utilization, fixed_context_tokens); + let budget = (effective_cap as f64 * utilization) as u64; // Auto-compact pass before the destructive prune. Headless // sessions don't get a `/compact` UI, so this is the only // way summaries get generated outside of an interactive @@ -886,7 +887,7 @@ pub async fn run( match crate::compact::maybe_auto_compact( &mut messages, effective_cap, - fixed_context_tokens, + preflight_tokens, provider.as_ref(), &model, ) @@ -915,10 +916,18 @@ pub async fn run( } } } - if context::prune_to_budget(&mut messages, budget) > 0 { + if context::prune_to_input_budget( + &mut messages, + budget, + fixed_context_tokens, + &input_tokens, + ) > 0 + { reset_context_serve(&tool_ctx); } + let mut sent_heuristic_tokens = + fixed_context_tokens.saturating_add(context::estimate_tokens(&messages) as u64); let mut sink = MessageSink { emit: &emit }; let turn = match crate::provider_watchdog::complete( provider.as_ref(), @@ -945,14 +954,18 @@ pub async fn run( effective_cap, now_ms, ); - let retry_budget = context::message_budget( - new_limit, - context::UTILIZATION, + let retry_budget = (new_limit as f64 * context::UTILIZATION) as u64; + if context::prune_to_input_budget( + &mut messages, + retry_budget, fixed_context_tokens, - ); - if context::prune_to_budget(&mut messages, retry_budget) > 0 { + &input_tokens, + ) > 0 + { reset_context_serve(&tool_ctx); } + sent_heuristic_tokens = fixed_context_tokens + .saturating_add(context::estimate_tokens(&messages) as u64); emit.emit(SessionEvent::Status { state: SessionState::Running, detail: Some(format!( @@ -993,6 +1006,8 @@ pub async fn run( } }; + input_tokens.record(turn.usage.input_tokens, sent_heuristic_tokens); + // Record the successful call so probe state advances // (and the learned limit grows on a probe that pushed // past the prior cap). @@ -1007,16 +1022,16 @@ pub async fn run( emit.emit(SessionEvent::Cost { usd: turn.usage.usd, - tokens_in: turn.usage.input_tokens, + tokens_in: turn.usage.input_tokens_or_zero(), tokens_out: turn.usage.output_tokens, tokens_cached: turn.usage.cached_tokens, model: Some(current_model_spec.clone()), }); // Context gauge (spec 0104): prefer a provider-reported runtime // allocation; otherwise retain Smith's learned/static fallback. - if turn.usage.input_tokens > 0 { + if let Some(reported_input_tokens) = turn.usage.input_tokens { emit.emit(SessionEvent::ContextUsage { - used_tokens: turn.usage.input_tokens, + used_tokens: reported_input_tokens, window_tokens: Some(effective_cap), }); // Per-component detail behind the gauge (spec 0156) — all @@ -1838,8 +1853,13 @@ fn build_profile_model( profile_api_key(profile, name, &["OPENAI_API_KEY"])?, )?), provider::routing::Provider::Anthropic => { - Box::new(provider::anthropic::Anthropic::with_config( + Box::new(provider::anthropic::Anthropic::with_options( base_url, + provider::anthropic::AnthropicOptions { + cache_control: profile.anthropic_cache_control, + betas: profile.anthropic_betas.clone(), + context_window_tokens: profile.anthropic_context_window_tokens, + }, profile_api_key(profile, name, &["ANTHROPIC_API_KEY"])?, )?) } @@ -2016,6 +2036,9 @@ mod tests { api_key: Some("test-key".to_string()), api_key_env: None, model: model.map(str::to_string), + anthropic_cache_control: None, + anthropic_betas: Vec::new(), + anthropic_context_window_tokens: None, } } diff --git a/crates/adapter-smith/src/compact.rs b/crates/adapter-smith/src/compact.rs index 436a1d68..cc5c3b64 100644 --- a/crates/adapter-smith/src/compact.rs +++ b/crates/adapter-smith/src/compact.rs @@ -165,19 +165,18 @@ pub async fn compact( /// the rolling-prune path. /// /// `effective_cap` is the per-model input-token cap the caller is using for -/// budget math. `fixed_tokens` is the system/tool prefix paid on every call. -/// The trigger compares their sum with the whole-window threshold so a large -/// fixed prefix cannot make Smith believe the context is emptier than it is. +/// budget math. `preflight_input_tokens` is provider-anchored when the prior +/// response reported usage and otherwise is Smith's full char heuristic. The +/// trigger never re-estimates that authoritative baseline here. pub async fn maybe_auto_compact( messages: &mut Vec, effective_cap: u64, - fixed_tokens: u64, + preflight_input_tokens: u64, provider: &dyn LlmProvider, model: &str, ) -> Result> { - let est = fixed_tokens.saturating_add(context::estimate_tokens(messages) as u64); let trigger = ((effective_cap as f64) * AUTO_COMPACT_RATIO) as u64; - if est < trigger { + if preflight_input_tokens < trigger { return Ok(None); } compact(messages, DEFAULT_KEEP_PAIRS, provider, model).await @@ -619,7 +618,8 @@ mod tests { let provider = StubProvider::new("unused"); // cap = 100k tokens; conversation is ~10 tokens; way under // threshold. - let outcome = maybe_auto_compact(&mut messages, 100_000, 0, &provider, "stub") + let preflight = context::estimate_tokens(&messages) as u64; + let outcome = maybe_auto_compact(&mut messages, 100_000, preflight, &provider, "stub") .await .unwrap(); assert!(outcome.is_none()); @@ -640,7 +640,8 @@ mod tests { messages.push(user("recent")); messages.push(asst("latest")); let provider = StubProvider::new("auto-summary"); - let outcome = maybe_auto_compact(&mut messages, 1000, 0, &provider, "stub") + let preflight = context::estimate_tokens(&messages) as u64; + let outcome = maybe_auto_compact(&mut messages, 1000, preflight, &provider, "stub") .await .unwrap() .expect("should auto-compact"); @@ -670,10 +671,16 @@ mod tests { assert!(est < prune_budget); let provider = StubProvider::new("pre-prune-summary"); - let outcome = maybe_auto_compact(&mut messages, effective_cap as u64, 0, &provider, "stub") - .await - .unwrap() - .expect("should auto-compact before rolling prune would fire"); + let outcome = maybe_auto_compact( + &mut messages, + effective_cap as u64, + est as u64, + &provider, + "stub", + ) + .await + .unwrap() + .expect("should auto-compact before rolling prune would fire"); assert!(outcome.dropped_turn_pairs > 0); } @@ -692,10 +699,39 @@ mod tests { assert!(message_tokens < (effective_cap as f64 * AUTO_COMPACT_RATIO) as u64); let provider = StubProvider::new("fixed-overhead-summary"); - let outcome = maybe_auto_compact(&mut messages, effective_cap, 6_000, &provider, "stub") + let outcome = maybe_auto_compact( + &mut messages, + effective_cap, + message_tokens + 6_000, + &provider, + "stub", + ) + .await + .unwrap() + .expect("fixed overhead should push the total over the trigger"); + assert!(outcome.dropped_turn_pairs > 0); + } + + #[tokio::test] + async fn auto_compact_obeys_provider_anchored_pressure() { + let mut messages = vec![ + user("one"), + asst("one"), + user("two"), + asst("two"), + user("three"), + asst("three"), + user("four"), + asst("four"), + user("five"), + asst("five"), + ]; + assert!(context::estimate_tokens(&messages) < 100); + let provider = StubProvider::new("provider-pressure-summary"); + let outcome = maybe_auto_compact(&mut messages, 1_000, 700, &provider, "stub") .await .unwrap() - .expect("fixed overhead should push the total over the trigger"); + .expect("provider count above 65% should trigger compaction"); assert!(outcome.dropped_turn_pairs > 0); } } diff --git a/crates/adapter-smith/src/context.rs b/crates/adapter-smith/src/context.rs index fb5de325..f9cb5910 100644 --- a/crates/adapter-smith/src/context.rs +++ b/crates/adapter-smith/src/context.rs @@ -1,13 +1,12 @@ //! Rolling-window context manager. //! -//! Estimates token count with a coarse `chars / 3.5` heuristic + a -//! safety margin, then prunes complete turn pairs (user → assistant → -//! tool exchanges between them) from the oldest end when the budget -//! is exceeded. The system prompt is owned by the caller and not -//! included here; we always keep the most-recent N turns. -//! -//! Approximate by design. v2 can swap in a real tokenizer (`tiktoken`, -//! `tokenizers`) and provider-native prompt caching. +//! Provider-reported input usage anchors context pressure after every +//! successful response. Between calls, a coarse `chars / 3.5` delta estimates +//! additions/removals for preflight pruning; before the first usage report (or +//! when a provider omits usage), the same heuristic estimates the whole input. +//! Complete turn pairs are pruned from the oldest end when the budget is +//! exceeded. The system prompt is owned by the caller; we always keep the +//! most-recent N turns. use crate::provider::{Content, Message, Role}; @@ -22,10 +21,9 @@ use crate::provider::{Content, Message, Role}; /// * OpenAI o-series (o1/o3/o4): 200K input. /// * Anthropic Claude 4.x Sonnet has a 1M-context tier available /// *only* with the `anthropic-beta: context-1m-2025-08-07` -/// header. Without that header it's 200K — and the current -/// `provider/anthropic.rs` does not send the header. So the -/// 200K value here matches what the wire actually allows. -/// Opus and Haiku stay at 200K regardless. +/// header. The first-party Anthropic provider advertises its effective 1M +/// allocation at runtime when it adds that beta; this conservative table +/// remains the fallback for compatible endpoints and other Claude models. pub fn context_window_tokens(provider: &str, model: &str) -> usize { match (provider, model) { ("openai", m) if m.starts_with("gpt-5") => 400_000, @@ -69,6 +67,59 @@ pub fn context_window_tokens(provider: &str, model: &str) -> usize { pub const UTILIZATION: f64 = 0.7; const MIN_KEEP_TURNS: usize = 2; +/// Session-local input-token accounting. +/// +/// A provider report applies to the exact request that produced it. We retain +/// the char estimate for that request only as a coordinate: the next +/// preflight count starts from the provider's authoritative number and adds +/// or subtracts the estimated content delta. This avoids replacing a real +/// tokenizer result with Smith's heuristic while still accounting for new +/// user/tool messages that have not reached the provider yet. +#[derive(Debug, Default, Clone, Copy)] +pub struct InputTokenCounter { + anchor: Option, +} + +#[derive(Debug, Clone, Copy)] +struct InputTokenAnchor { + reported: u64, + heuristic_at_request: u64, +} + +impl InputTokenCounter { + /// Record usage for the exact request whose full-input heuristic was + /// `heuristic_at_request`. Missing usage clears any stale provider anchor, + /// making subsequent calls use the documented full-estimate fallback. + pub fn record(&mut self, reported: Option, heuristic_at_request: u64) { + self.anchor = reported.map(|reported| InputTokenAnchor { + reported, + heuristic_at_request, + }); + } + + /// Estimate the next request from the latest provider count plus only the + /// char-estimated delta since that report. With no report, return the full + /// heuristic unchanged. + pub fn preflight(&self, current_heuristic: u64) -> u64 { + let Some(anchor) = self.anchor else { + return current_heuristic; + }; + if current_heuristic >= anchor.heuristic_at_request { + anchor + .reported + .saturating_add(current_heuristic - anchor.heuristic_at_request) + } else { + anchor + .reported + .saturating_sub(anchor.heuristic_at_request - current_heuristic) + } + } + + pub fn reset(&mut self) { + self.anchor = None; + } +} + /// Char-heuristic token estimate for a raw string — the same `chars / 3.5` /// rule as [`estimate_tokens`], for prompt sections that aren't `Message`s. pub fn estimate_tokens_str(s: &str) -> u64 { @@ -175,11 +226,54 @@ fn estimate_tool_tokens(tool_specs: &[crate::provider::ToolSpec]) -> u64 { /// Convert a whole-window utilization target into the portion available to /// conversation messages after the fixed prompt and tool overhead is paid. +#[cfg(test)] pub fn message_budget(window_tokens: u64, utilization: f64, fixed_tokens: u64) -> usize { let total_budget = (window_tokens as f64 * utilization) as u64; usize::try_from(total_budget.saturating_sub(fixed_tokens)).unwrap_or(usize::MAX) } +/// Full preflight input estimate (fixed prompt/tools plus conversation), +/// anchored to provider usage when available. +pub fn preflight_input_tokens( + counter: &InputTokenCounter, + fixed_tokens: u64, + messages: &[Message], +) -> u64 { + let heuristic = fixed_tokens.saturating_add(estimate_tokens(messages) as u64); + counter.preflight(heuristic) +} + +/// Prune oldest turn pairs against a whole-input budget. Unlike +/// [`prune_to_budget`], this includes fixed prompt/tool overhead and uses the +/// latest provider report as its baseline when one exists. +pub fn prune_to_input_budget( + messages: &mut Vec, + budget: u64, + fixed_tokens: u64, + counter: &InputTokenCounter, +) -> usize { + let mut pruned = 0; + while preflight_input_tokens(counter, fixed_tokens, messages) > budget { + let mut user_seen = 0; + let can_prune = messages.iter().any(|m| { + if matches!(m.role, Role::User) { + user_seen += 1; + } + user_seen == MIN_KEEP_TURNS + 1 + }); + if !can_prune { + break; + } + let cut = find_first_user_run_end(messages); + if cut == 0 { + break; + } + messages.drain(..cut); + pruned += 1; + } + pruned +} + /// Rough token estimate (chars / 3.5). Safe to overestimate. pub fn estimate_tokens(messages: &[Message]) -> usize { let mut chars = 0usize; @@ -227,10 +321,10 @@ pub fn prune(messages: &mut Vec, provider: &str, model: &str) -> usize prune_to_budget(messages, cap) } -/// Variant of `prune` that takes an explicit token budget instead -/// of looking up the hardcoded table. Used by the learned-limit / -/// probe path in `agent.rs` so the budget reflects the per-model -/// runtime knowledge. +/// Legacy message-only variant retained to exercise heuristic fallback pruning +/// in unit tests. Production paths use [`prune_to_input_budget`] so fixed +/// overhead and provider-reported usage participate in the decision. +#[cfg(test)] pub fn prune_to_budget(messages: &mut Vec, cap: usize) -> usize { let mut pruned = 0; while estimate_tokens(messages) > cap { @@ -435,6 +529,47 @@ mod tests { assert_eq!(message_budget(8_000, 0.7, 20_000), 0); } + #[test] + fn provider_report_anchors_only_the_preflight_delta() { + let mut counter = InputTokenCounter::default(); + counter.record(Some(12_000), 10_000); + assert_eq!(counter.preflight(10_700), 12_700); + assert_eq!(counter.preflight(8_500), 10_500); + } + + #[test] + fn missing_provider_usage_falls_back_to_full_heuristic() { + let mut counter = InputTokenCounter::default(); + assert_eq!(counter.preflight(9_000), 9_000); + counter.record(Some(12_000), 10_000); + counter.record(None, 11_000); + assert_eq!(counter.preflight(9_000), 9_000); + } + + #[test] + fn input_budget_pruning_uses_provider_anchored_pressure() { + let large = "x".repeat(10_000); + let mut messages = vec![ + user(&large), + asst(&large), + user("middle"), + asst("answer"), + user("latest"), + asst("answer"), + ]; + let heuristic = 1_000 + estimate_tokens(&messages) as u64; + let mut counter = InputTokenCounter::default(); + // Provider tokenizer says this exact request is much larger than the + // char heuristic. A 9K budget must therefore prune the oldest turn. + counter.record(Some(12_000), heuristic); + assert_eq!( + prune_to_input_budget(&mut messages, 9_000, 1_000, &counter), + 1 + ); + assert!(matches!(messages.first().map(|m| m.role), Some(Role::User))); + assert_eq!(messages.len(), 4); + } + #[test] fn grok_4_6_gets_its_advertised_context_window_on_both_auth_paths() { assert_eq!(context_window_tokens("grok", "grok-4.6"), 500_000); diff --git a/crates/adapter-smith/src/interactive.rs b/crates/adapter-smith/src/interactive.rs index ec62fa5c..524b834f 100644 --- a/crates/adapter-smith/src/interactive.rs +++ b/crates/adapter-smith/src/interactive.rs @@ -1987,6 +1987,7 @@ pub async fn run( let fixed_context_tokens = prompt_sections.fixed_tokens(&specs); let mut provider_context_window = None; let mut provider_context_checked = false; + let mut input_tokens = context::InputTokenCounter::default(); let term = Terminal::new(&emit); let resuming = persist::is_resume(); @@ -2380,6 +2381,7 @@ pub async fn run( model = new_model; provider_context_window = None; provider_context_checked = false; + input_tokens.reset(); term.note(&format!("(model → {}:{})", display_name, model)); emit.emit(SessionEvent::Status { state: SessionState::Running, @@ -2397,6 +2399,7 @@ pub async fn run( } CommandId::Reset => { messages.clear(); + input_tokens.reset(); if let Some(p) = persist.as_mut() { p.reset(); } @@ -2575,6 +2578,7 @@ pub async fn run( provider = resolved.provider; provider_context_window = None; provider_context_checked = false; + input_tokens.reset(); model_ready = true; term.note(&format!("(model configured: {display_name}:{model})")); emit.emit(SessionEvent::Status { @@ -2669,11 +2673,11 @@ pub async fn run( } let hardcoded_cap = context::context_window_tokens(provider_name, &model); let learned = limits.get(provider_name, &model); - let est = - fixed_context_tokens.saturating_add(context::estimate_tokens(&messages) as u64); + let preflight_tokens = + context::preflight_input_tokens(&input_tokens, fixed_context_tokens, &messages); let is_probe = provider_context_window.is_none() && learned.is_some() - && limits.should_probe(provider_name, &model, est, now_ms); + && limits.should_probe(provider_name, &model, preflight_tokens, now_ms); let effective_cap = provider_context_window .or(learned) .unwrap_or(hardcoded_cap as u64); @@ -2682,7 +2686,7 @@ pub async fn run( } else { context::UTILIZATION }; - let budget = context::message_budget(effective_cap, utilization, fixed_context_tokens); + let budget = (effective_cap as f64 * utilization) as u64; // Auto-compact pass before the destructive rolling prune. // We try this first so historical context survives as a // summary instead of vanishing. On any failure (provider @@ -2693,7 +2697,7 @@ pub async fn run( match crate::compact::maybe_auto_compact( &mut messages, effective_cap, - fixed_context_tokens, + preflight_tokens, provider.as_ref(), &model, ) @@ -2724,9 +2728,17 @@ pub async fn run( } } } - if context::prune_to_budget(&mut messages, budget) > 0 { + if context::prune_to_input_budget( + &mut messages, + budget, + fixed_context_tokens, + &input_tokens, + ) > 0 + { crate::agent::reset_context_serve(&tool_ctx); } + let mut sent_heuristic_tokens = + fixed_context_tokens.saturating_add(context::estimate_tokens(&messages) as u64); let mut sink = PtySink::new(&emit, pty_width, turn_started_at_ms); // Wrap the provider call so user typing during the // stream is fed to the editor and pressed-Enter lines @@ -2774,14 +2786,18 @@ pub async fn run( effective_cap, now_ms, ); - let retry_budget = context::message_budget( - new_limit, - context::UTILIZATION, + let retry_budget = (new_limit as f64 * context::UTILIZATION) as u64; + if context::prune_to_input_budget( + &mut messages, + retry_budget, fixed_context_tokens, - ); - if context::prune_to_budget(&mut messages, retry_budget) > 0 { + &input_tokens, + ) > 0 + { crate::agent::reset_context_serve(&tool_ctx); } + sent_heuristic_tokens = fixed_context_tokens + .saturating_add(context::estimate_tokens(&messages) as u64); term.note(&format!( "(context overflow — relearned cap as {} tokens, retrying)", new_limit @@ -2870,6 +2886,7 @@ pub async fn run( break; } }; + input_tokens.record(turn.usage.input_tokens, sent_heuristic_tokens); // Record the call so probe state advances (and the learned // limit grows on a probe that pushed past the prior cap). limits.record_call( @@ -2882,16 +2899,16 @@ pub async fn run( ); emit.emit(SessionEvent::Cost { usd: turn.usage.usd, - tokens_in: turn.usage.input_tokens, + tokens_in: turn.usage.input_tokens_or_zero(), tokens_out: turn.usage.output_tokens, tokens_cached: turn.usage.cached_tokens, model: current_model_spec.clone(), }); // Context gauge (spec 0104): prefer a provider-reported runtime // allocation; otherwise retain Smith's learned/static fallback. - if turn.usage.input_tokens > 0 { + if let Some(reported_input_tokens) = turn.usage.input_tokens { emit.emit(SessionEvent::ContextUsage { - used_tokens: turn.usage.input_tokens, + used_tokens: reported_input_tokens, window_tokens: Some(effective_cap), }); // Per-component detail behind the gauge (spec 0156) — all diff --git a/crates/adapter-smith/src/model_limits.rs b/crates/adapter-smith/src/model_limits.rs index 77328df9..6e68acec 100644 --- a/crates/adapter-smith/src/model_limits.rs +++ b/crates/adapter-smith/src/model_limits.rs @@ -166,17 +166,15 @@ impl ModelLimits { new_limit } - /// Called after a successful provider call. `actual_input_tokens` - /// is what the provider reported in its usage block; far more - /// accurate than our chars/3.5 estimate. If this call was a - /// probe AND the actual usage exceeded the prior learned limit, - /// bump the learned limit to `actual + 5%` so subsequent calls - /// can use the headroom. + /// Called after a successful provider call. `actual_input_tokens` is the + /// provider's authoritative usage when present. If this call was a probe + /// and that usage exceeded the prior learned limit, bump the limit to + /// `actual + 5%`; missing usage never manufactures a successful probe. pub fn record_call( &mut self, provider: &str, model: &str, - actual_input_tokens: u64, + actual_input_tokens: Option, was_probe: bool, fallback: u64, now_ms: i64, @@ -194,8 +192,10 @@ impl ModelLimits { entry.calls_since_probe = 0; // Bump only if the probe actually pushed past the prior // limit — otherwise the probe didn't test anything. - if actual_input_tokens > entry.learned_input_tokens { - entry.learned_input_tokens = ((actual_input_tokens as f64) * 1.05) as u64; + if let Some(actual_input_tokens) = actual_input_tokens { + if actual_input_tokens > entry.learned_input_tokens { + entry.learned_input_tokens = ((actual_input_tokens as f64) * 1.05) as u64; + } } } else { entry.calls_since_probe = entry.calls_since_probe.saturating_add(1); @@ -242,13 +242,21 @@ mod tests { let mut s = ModelLimits::default(); s.record_overflow("openai", "gpt-5", Some(400_000), 400_000, 0); // Probe that didn't actually push past 400K → no bump. - s.record_call("openai", "gpt-5", 380_000, true, 400_000, 1_000); + s.record_call("openai", "gpt-5", Some(380_000), true, 400_000, 1_000); assert_eq!(s.get("openai", "gpt-5"), Some(400_000)); // Probe that pushed to 450K → bump to 450K * 1.05. - s.record_call("openai", "gpt-5", 450_000, true, 400_000, 2_000); + s.record_call("openai", "gpt-5", Some(450_000), true, 400_000, 2_000); assert_eq!(s.get("openai", "gpt-5"), Some((450_000.0 * 1.05) as u64)); } + #[test] + fn missing_probe_usage_does_not_raise_learned_limit() { + let mut s = ModelLimits::default(); + s.record_overflow("openai", "gpt-5", Some(400_000), 400_000, 0); + s.record_call("openai", "gpt-5", None, true, 400_000, 1_000); + assert_eq!(s.get("openai", "gpt-5"), Some(400_000)); + } + /// Round-trip via the same JSON shape the daemon stores on disk. /// Catches accidental serde-rename or required-field churn that /// would silently break learned limits across daemon restarts. diff --git a/crates/adapter-smith/src/provider/anthropic.rs b/crates/adapter-smith/src/provider/anthropic.rs index e257237b..36468f64 100644 --- a/crates/adapter-smith/src/provider/anthropic.rs +++ b/crates/adapter-smith/src/provider/anthropic.rs @@ -19,34 +19,152 @@ use eventsource_stream::Eventsource; use futures::StreamExt; use serde_json::{json, Value}; +pub(crate) const CONTEXT_1M_BETA: &str = "context-1m-2025-08-07"; +const CONTEXT_1M_TOKENS: u64 = 1_000_000; +const DEFAULT_CONTEXT_TOKENS: u64 = 200_000; +const MAX_CACHE_BREAKPOINTS: usize = 4; + +/// Capabilities for an Anthropic endpoint. Named profiles populate these +/// fields directly; the built-in provider reads equivalent environment +/// overrides (documented in `docs/smith.md`). +#[derive(Debug, Clone, Default)] +pub struct AnthropicOptions { + pub cache_control: Option, + pub betas: Vec, + pub context_window_tokens: Option, +} + pub struct Anthropic { client: reqwest::Client, base_url: String, api_key: String, + cache_control: bool, + betas: Vec, + context_window_tokens: Option, + first_party: bool, } impl Anthropic { pub fn from_env() -> Result { let api_key = std::env::var("ANTHROPIC_API_KEY").map_err(|_| anyhow!("ANTHROPIC_API_KEY not set"))?; - Self::with_config(std::env::var("ANTHROPIC_BASE_URL").ok(), api_key) + Self::with_options( + std::env::var("ANTHROPIC_BASE_URL").ok(), + options_from_env()?, + api_key, + ) } /// Build with an explicit base URL (None → public Anthropic) and key. /// Used by named `[smith.models.*]` profiles. + #[cfg(test)] pub fn with_config(base_url: Option, api_key: String) -> Result { + Self::with_options(base_url, AnthropicOptions::default(), api_key) + } + + /// Build with endpoint capabilities supplied by a named model profile. + /// Options precede the key so call sites keep credentials visually last. + pub fn with_options( + base_url: Option, + options: AnthropicOptions, + api_key: String, + ) -> Result { + if options.context_window_tokens == Some(0) { + anyhow::bail!("Anthropic context window must be greater than zero"); + } let base_url = base_url .unwrap_or_else(|| "https://api.anthropic.com/v1".to_string()) .trim_end_matches('/') .to_string(); + let first_party = is_first_party_endpoint(&base_url); Ok(Self { client: reqwest::Client::builder() .build() .context("build reqwest client")?, base_url, api_key, + cache_control: options.cache_control.unwrap_or(first_party), + betas: dedup_betas(options.betas), + context_window_tokens: options.context_window_tokens, + first_party, }) } + + fn request_betas(&self, model: &str) -> Vec { + let mut betas = self.betas.clone(); + let explicitly_capped_at_default = self + .context_window_tokens + .is_some_and(|tokens| tokens <= DEFAULT_CONTEXT_TOKENS); + if self.first_party + && supports_context_1m_beta(model) + && !explicitly_capped_at_default + && !betas.iter().any(|beta| beta == CONTEXT_1M_BETA) + { + betas.push(CONTEXT_1M_BETA.to_string()); + } + betas + } + + fn context_window_for_model(&self, model: &str) -> Option { + self.context_window_tokens.or_else(|| { + self.request_betas(model) + .iter() + .any(|beta| beta == CONTEXT_1M_BETA) + .then_some(CONTEXT_1M_TOKENS) + }) + } +} + +fn supports_context_1m_beta(model: &str) -> bool { + model.to_ascii_lowercase().contains("claude-sonnet-4") +} + +fn is_first_party_endpoint(base_url: &str) -> bool { + reqwest::Url::parse(base_url) + .ok() + .and_then(|url| url.host_str().map(str::to_owned)) + .is_some_and(|host| host.eq_ignore_ascii_case("api.anthropic.com")) +} + +fn dedup_betas(betas: Vec) -> Vec { + let mut out = Vec::new(); + for beta in betas { + let beta = beta.trim(); + if !beta.is_empty() && !out.iter().any(|existing| existing == beta) { + out.push(beta.to_string()); + } + } + out +} + +fn options_from_env() -> Result { + let cache_control = match std::env::var("CONSTRUCT_SMITH_ANTHROPIC_CACHE_CONTROL") { + Ok(value) => match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "on" | "yes" => Some(true), + "0" | "false" | "off" | "no" => Some(false), + other => anyhow::bail!( + "CONSTRUCT_SMITH_ANTHROPIC_CACHE_CONTROL must be on/off (got `{other}`)" + ), + }, + Err(_) => None, + }; + let betas = std::env::var("CONSTRUCT_SMITH_ANTHROPIC_BETAS") + .ok() + .map(|value| value.split(',').map(str::to_string).collect()) + .unwrap_or_default(); + let context_window_tokens = std::env::var("CONSTRUCT_SMITH_ANTHROPIC_CONTEXT_WINDOW_TOKENS") + .ok() + .map(|value| { + value.parse::().with_context(|| { + "CONSTRUCT_SMITH_ANTHROPIC_CONTEXT_WINDOW_TOKENS must be a positive integer" + }) + }) + .transpose()?; + Ok(AnthropicOptions { + cache_control, + betas, + context_window_tokens, + }) } pub(crate) fn messages_to_anthropic(messages: &[Message]) -> Vec { @@ -139,6 +257,99 @@ pub(crate) fn tools_to_anthropic(tools: &[ToolSpec]) -> Vec { .collect() } +/// Add up to Anthropic's four ephemeral cache breakpoints in stable-prefix +/// order: system, tool definitions, then the latest user-message boundaries. +/// This mutates an already valid Messages request so the same policy works for +/// API-key and Claude OAuth requests while Anthropic-compatible providers can +/// continue using the unmodified wire helpers. +pub(crate) fn apply_cache_control(body: &mut Value) { + let Some(object) = body.as_object_mut() else { + return; + }; + let mut remaining = MAX_CACHE_BREAKPOINTS; + + if remaining > 0 { + if let Some(system) = object.get_mut("system") { + if mark_content_tail(system) { + remaining -= 1; + } + } + } + if remaining > 0 { + if let Some(last_tool) = object + .get_mut("tools") + .and_then(Value::as_array_mut) + .and_then(|tools| tools.last_mut()) + { + if mark_object(last_tool) { + remaining -= 1; + } + } + } + if remaining == 0 { + return; + } + let Some(messages) = object.get_mut("messages").and_then(Value::as_array_mut) else { + return; + }; + for message in messages.iter_mut().rev() { + if remaining == 0 { + break; + } + if message.get("role").and_then(Value::as_str) != Some("user") { + continue; + } + if message.get_mut("content").is_some_and(mark_content_tail) { + remaining -= 1; + } + } +} + +fn mark_content_tail(content: &mut Value) -> bool { + if let Some(text) = content.as_str().map(str::to_owned) { + *content = json!([{ + "type": "text", + "text": text, + "cache_control": { "type": "ephemeral" }, + }]); + return true; + } + content + .as_array_mut() + .and_then(|blocks| blocks.last_mut()) + .is_some_and(mark_object) +} + +fn mark_object(value: &mut Value) -> bool { + let Some(object) = value.as_object_mut() else { + return false; + }; + object.insert("cache_control".to_string(), json!({ "type": "ephemeral" })); + true +} + +/// Anthropic reports uncached, cache-write, and cache-read prompt tokens as +/// separate fields. Their sum is what occupied the model's input window; only +/// the read portion is the cached-token subset shown in cost telemetry. +fn update_input_usage(usage: &mut Usage, value: &Value) { + let fresh = value.get("input_tokens").and_then(Value::as_u64); + let created = value + .get("cache_creation_input_tokens") + .and_then(Value::as_u64); + let read = value.get("cache_read_input_tokens").and_then(Value::as_u64); + if fresh.is_some() || created.is_some() || read.is_some() { + usage.input_tokens = Some( + fresh + .unwrap_or(0) + .saturating_add(created.unwrap_or(0)) + .saturating_add(read.unwrap_or(0)), + ); + } + if let Some(read) = read { + usage.cached_tokens = read; + } +} + /// Shared handler for an Anthropic Messages API streaming response: checks /// the HTTP status (mapping context-overflow 400s to [`super::ContextOverflow`] /// so the agent loop's learn-and-retry path can fire), then parses the typed @@ -189,12 +400,7 @@ pub(crate) async fn read_message_stream( match ty { "message_start" => { if let Some(u) = v.pointer("/message/usage") { - usage.input_tokens = - u.get("input_tokens").and_then(|n| n.as_u64()).unwrap_or(0); - usage.cached_tokens = u - .get("cache_read_input_tokens") - .and_then(|n| n.as_u64()) - .unwrap_or(0); + update_input_usage(&mut usage, u); } } "content_block_start" => { @@ -324,6 +530,10 @@ impl LlmProvider for Anthropic { true } + async fn effective_context_window_tokens(&self, model: &str) -> Option { + self.context_window_for_model(model) + } + async fn complete( &self, model: &str, @@ -344,13 +554,21 @@ impl LlmProvider for Anthropic { if !tools.is_empty() { body["tools"] = Value::Array(tools_to_anthropic(tools)); } + if self.cache_control { + apply_cache_control(&mut body); + } let url = format!("{}/messages", self.base_url); - let resp = self + let mut request = self .client .post(&url) .header("x-api-key", &self.api_key) - .header("anthropic-version", "2023-06-01") + .header("anthropic-version", "2023-06-01"); + let betas = self.request_betas(model); + if !betas.is_empty() { + request = request.header("anthropic-beta", betas.join(",")); + } + let resp = request .json(&body) .send() .await @@ -404,4 +622,140 @@ mod tests { assert_eq!(wire[0]["content"][1]["source"]["media_type"], "image/jpeg"); assert_eq!(wire[0]["content"][1]["source"]["data"], "YWJj"); } + + fn test_message(role: Role, text: &str) -> Message { + Message { + role, + content: Content::Text { + text: text.to_string(), + }, + } + } + + #[test] + fn usage_sums_fresh_cache_write_and_cache_read_tokens() { + let mut usage = Usage::default(); + update_input_usage( + &mut usage, + &json!({ + "input_tokens": 101, + "cache_creation_input_tokens": 2_000, + "cache_read_input_tokens": 30_000, + }), + ); + assert_eq!(usage.input_tokens, Some(32_101)); + assert_eq!(usage.cached_tokens, 30_000); + } + + #[test] + fn cache_control_marks_four_stable_prefix_breakpoints() { + let messages = vec![ + test_message(Role::User, "one"), + test_message(Role::Assistant, "answer"), + test_message(Role::User, "two"), + test_message(Role::Assistant, "answer"), + test_message(Role::User, "three"), + ]; + let tools = vec![ToolSpec { + name: "shell".into(), + description: "run".into(), + schema: json!({"type": "object"}), + }]; + let mut body = json!({ + "system": "stable system", + "tools": tools_to_anthropic(&tools), + "messages": messages_to_anthropic(&messages), + }); + apply_cache_control(&mut body); + + fn count(value: &Value) -> usize { + match value { + Value::Array(values) => values.iter().map(count).sum(), + Value::Object(map) => { + usize::from(map.contains_key("cache_control")) + + map.values().map(count).sum::() + } + _ => 0, + } + } + assert_eq!(count(&body), MAX_CACHE_BREAKPOINTS); + assert_eq!( + body.pointer("/system/0/cache_control/type") + .and_then(Value::as_str), + Some("ephemeral") + ); + assert_eq!( + body.pointer("/tools/0/cache_control/type") + .and_then(Value::as_str), + Some("ephemeral") + ); + // With system + tools consuming two slots, the newest two user + // boundaries are cached and the oldest remains untouched. + assert!(body.pointer("/messages/0/content").unwrap().is_string()); + assert_eq!( + body.pointer("/messages/2/content/0/cache_control/type") + .and_then(Value::as_str), + Some("ephemeral") + ); + assert_eq!( + body.pointer("/messages/4/content/0/cache_control/type") + .and_then(Value::as_str), + Some("ephemeral") + ); + } + + #[tokio::test] + async fn first_party_sonnet_enables_1m_beta_and_window() { + let provider = Anthropic::with_config(None, "test".into()).unwrap(); + assert_eq!( + provider + .effective_context_window_tokens("claude-sonnet-4-6") + .await, + Some(CONTEXT_1M_TOKENS) + ); + assert_eq!( + provider.request_betas("claude-sonnet-4-6"), + [CONTEXT_1M_BETA] + ); + } + + #[tokio::test] + async fn compatible_endpoint_requires_explicit_capabilities() { + let provider = + Anthropic::with_config(Some("https://gateway.example/v1".into()), "test".into()) + .unwrap(); + assert!(!provider.cache_control); + assert!(provider.request_betas("claude-sonnet-4-6").is_empty()); + assert_eq!( + provider + .effective_context_window_tokens("claude-sonnet-4-6") + .await, + None + ); + } + + #[tokio::test] + async fn compatible_endpoint_accepts_declared_capabilities() { + let provider = Anthropic::with_options( + Some("https://gateway.example/v1".into()), + AnthropicOptions { + cache_control: Some(true), + betas: vec!["gateway-long-context".into(), "gateway-long-context".into()], + context_window_tokens: Some(750_000), + }, + "test".into(), + ) + .unwrap(); + assert!(provider.cache_control); + assert_eq!( + provider.request_betas("vendor-model"), + ["gateway-long-context"] + ); + assert_eq!( + provider + .effective_context_window_tokens("vendor-model") + .await, + Some(750_000) + ); + } } diff --git a/crates/adapter-smith/src/provider/antigravity_oauth.rs b/crates/adapter-smith/src/provider/antigravity_oauth.rs index 3d10b257..0d40191a 100644 --- a/crates/adapter-smith/src/provider/antigravity_oauth.rs +++ b/crates/adapter-smith/src/provider/antigravity_oauth.rs @@ -260,7 +260,7 @@ impl LlmProvider for AntigravityOauth { } if let Some(u) = v.get("usageMetadata") { if let Some(n) = u.get("promptTokenCount").and_then(|n| n.as_u64()) { - usage.input_tokens = n; + usage.input_tokens = Some(n); } if let Some(n) = u.get("candidatesTokenCount").and_then(|n| n.as_u64()) { usage.output_tokens = n; diff --git a/crates/adapter-smith/src/provider/claude_oauth.rs b/crates/adapter-smith/src/provider/claude_oauth.rs index 85ed27e8..c92e5943 100644 --- a/crates/adapter-smith/src/provider/claude_oauth.rs +++ b/crates/adapter-smith/src/provider/claude_oauth.rs @@ -372,6 +372,10 @@ impl LlmProvider for ClaudeOauth { if !tools.is_empty() { body["tools"] = Value::Array(super::anthropic::tools_to_anthropic(tools)); } + // Claude Code's first-party endpoint supports Anthropic prompt-cache + // directives. Keep compatible providers (for example Kimi's shared + // wire helpers) unchanged by applying them only on this request path. + super::anthropic::apply_cache_control(&mut body); let resp = self .http diff --git a/crates/adapter-smith/src/provider/codex_oauth.rs b/crates/adapter-smith/src/provider/codex_oauth.rs index 9f1e1dc4..2468399e 100644 --- a/crates/adapter-smith/src/provider/codex_oauth.rs +++ b/crates/adapter-smith/src/provider/codex_oauth.rs @@ -515,10 +515,9 @@ impl CodexOauth { } "response.completed" | "response.incomplete" | "response.failed" => { if let Some(u) = chunk.pointer("/response/usage") { - usage.input_tokens = u - .get("input_tokens") - .and_then(|n| n.as_u64()) - .unwrap_or(usage.input_tokens); + if let Some(n) = u.get("input_tokens").and_then(|n| n.as_u64()) { + usage.input_tokens = Some(n); + } usage.output_tokens = u .get("output_tokens") .and_then(|n| n.as_u64()) @@ -1188,10 +1187,9 @@ impl LlmProvider for CodexOauth { "response.completed" | "response.incomplete" | "response.failed" => { terminal_event_seen = true; if let Some(u) = chunk.pointer("/response/usage") { - usage.input_tokens = u - .get("input_tokens") - .and_then(|n| n.as_u64()) - .unwrap_or(usage.input_tokens); + if let Some(n) = u.get("input_tokens").and_then(|n| n.as_u64()) { + usage.input_tokens = Some(n); + } usage.output_tokens = u .get("output_tokens") .and_then(|n| n.as_u64()) diff --git a/crates/adapter-smith/src/provider/config.rs b/crates/adapter-smith/src/provider/config.rs index e6a01251..363a41d6 100644 --- a/crates/adapter-smith/src/provider/config.rs +++ b/crates/adapter-smith/src/provider/config.rs @@ -55,6 +55,17 @@ pub struct ModelProfile { /// `@:`. #[serde(default)] pub model: Option, + /// Anthropic-only: override prompt-cache breakpoints. Public Anthropic + /// endpoints default on; compatible gateways default off. + #[serde(default)] + pub anthropic_cache_control: Option, + /// Anthropic-only beta capabilities to send in `anthropic-beta`. + #[serde(default)] + pub anthropic_betas: Vec, + /// Anthropic-only effective input window. Useful for newly released + /// large-context models and compatible gateways Smith cannot identify. + #[serde(default)] + pub anthropic_context_window_tokens: Option, } /// Only the `[smith]` table is deserialized; every other top-level key in @@ -118,6 +129,9 @@ mod tests { assert_eq!(p.api_key_env.as_deref(), Some("DEEPSEEK_API_KEY")); assert_eq!(p.model.as_deref(), Some("deepseek-chat")); assert!(p.api_key.is_none()); + assert!(p.anthropic_cache_control.is_none()); + assert!(p.anthropic_betas.is_empty()); + assert!(p.anthropic_context_window_tokens.is_none()); } #[test] @@ -172,4 +186,21 @@ mod tests { assert!(models.contains_key("deepseek")); assert!(models.contains_key("groq")); } + + #[test] + fn parses_anthropic_capabilities() { + let toml = r#" + [smith.models.claude-long] + provider = "anthropic" + model = "claude-sonnet-4-6" + anthropic_cache_control = true + anthropic_betas = ["context-1m-2025-08-07", "example-beta"] + anthropic_context_window_tokens = 1000000 + "#; + let models = parse(toml).expect("parse"); + let p = models.get("claude-long").expect("profile"); + assert_eq!(p.anthropic_cache_control, Some(true)); + assert_eq!(p.anthropic_betas, ["context-1m-2025-08-07", "example-beta"]); + assert_eq!(p.anthropic_context_window_tokens, Some(1_000_000)); + } } diff --git a/crates/adapter-smith/src/provider/gemini.rs b/crates/adapter-smith/src/provider/gemini.rs index 0bc7dbe6..c8c2981d 100644 --- a/crates/adapter-smith/src/provider/gemini.rs +++ b/crates/adapter-smith/src/provider/gemini.rs @@ -290,7 +290,7 @@ impl LlmProvider for Gemini { } if let Some(u) = v.get("usageMetadata") { if let Some(n) = u.get("promptTokenCount").and_then(|n| n.as_u64()) { - usage.input_tokens = n; + usage.input_tokens = Some(n); } if let Some(n) = u.get("candidatesTokenCount").and_then(|n| n.as_u64()) { usage.output_tokens = n; diff --git a/crates/adapter-smith/src/provider/meta.rs b/crates/adapter-smith/src/provider/meta.rs index f4e529a9..330700a3 100644 --- a/crates/adapter-smith/src/provider/meta.rs +++ b/crates/adapter-smith/src/provider/meta.rs @@ -291,10 +291,8 @@ impl LlmProvider for Meta { "response.completed" | "response.incomplete" | "response.failed" => { terminal_event_seen = true; if let Some(provider_usage) = chunk.pointer("/response/usage") { - usage.input_tokens = provider_usage - .get("input_tokens") - .and_then(Value::as_u64) - .unwrap_or_default(); + usage.input_tokens = + provider_usage.get("input_tokens").and_then(Value::as_u64); usage.output_tokens = provider_usage .get("output_tokens") .and_then(Value::as_u64) diff --git a/crates/adapter-smith/src/provider/mod.rs b/crates/adapter-smith/src/provider/mod.rs index 03b03ecb..e78fe256 100644 --- a/crates/adapter-smith/src/provider/mod.rs +++ b/crates/adapter-smith/src/provider/mod.rs @@ -162,13 +162,26 @@ pub struct ToolCall { #[derive(Debug, Clone, Copy, Default)] pub struct Usage { - pub input_tokens: u64, + /// Provider-reported prompt tokens. `None` means the provider omitted + /// usage; keep that distinct from a real zero so context budgeting can + /// fall back to an estimate deliberately rather than treating missing + /// telemetry as authoritative. + pub input_tokens: Option, pub output_tokens: u64, /// Cached input tokens (subset of `input_tokens`). 0 when unknown. pub cached_tokens: u64, pub usd: f64, } +impl Usage { + /// Value used by cumulative cost events, whose protocol predates the + /// explicit unknown state. Context-pressure decisions should use the + /// `Option` directly instead. + pub fn input_tokens_or_zero(self) -> u64 { + self.input_tokens.unwrap_or(0) + } +} + /// Why the provider stopped producing tokens. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum StopReason { diff --git a/crates/adapter-smith/src/provider/ollama.rs b/crates/adapter-smith/src/provider/ollama.rs index 52768016..8cb483ba 100644 --- a/crates/adapter-smith/src/provider/ollama.rs +++ b/crates/adapter-smith/src/provider/ollama.rs @@ -310,7 +310,7 @@ impl LlmProvider for Ollama { } if v.get("done").and_then(|b| b.as_bool()).unwrap_or(false) { if let Some(n) = v.get("prompt_eval_count").and_then(|n| n.as_u64()) { - usage.input_tokens = n; + usage.input_tokens = Some(n); } if let Some(n) = v.get("eval_count").and_then(|n| n.as_u64()) { usage.output_tokens = n; diff --git a/crates/adapter-smith/src/provider/openai.rs b/crates/adapter-smith/src/provider/openai.rs index bca3773e..ad15404c 100644 --- a/crates/adapter-smith/src/provider/openai.rs +++ b/crates/adapter-smith/src/provider/openai.rs @@ -256,10 +256,9 @@ impl LlmProvider for OpenAi { Err(_) => continue, }; if let Some(u) = chunk.get("usage") { - usage.input_tokens = u - .get("prompt_tokens") - .and_then(|n| n.as_u64()) - .unwrap_or(usage.input_tokens); + if let Some(n) = u.get("prompt_tokens").and_then(|n| n.as_u64()) { + usage.input_tokens = Some(n); + } usage.output_tokens = u .get("completion_tokens") .and_then(|n| n.as_u64()) diff --git a/crates/daemon/src/config.rs b/crates/daemon/src/config.rs index 4b2cfcca..c7b1a4da 100644 --- a/crates/daemon/src/config.rs +++ b/crates/daemon/src/config.rs @@ -280,6 +280,11 @@ enabled = true # provider = "anthropic" # api_key_env = "ANTHROPIC_API_KEY" # model = "claude-sonnet-4-6" +# # Public Anthropic defaults cache control on and enables the known Sonnet 4 +# # one-million-token beta automatically. Explicit endpoint capabilities: +# # anthropic_cache_control = true +# # anthropic_betas = ["context-1m-2025-08-07"] +# # anthropic_context_window_tokens = 1000000 # Gemini direct API: # [smith.models.gemini] @@ -363,6 +368,9 @@ enabled = true # CONSTRUCT_SMITH_AUTOMODE — set to "1" to enable autonomous mode by default # CONSTRUCT_SMITH_MAX_STEPS — max tool-call steps per turn # CONSTRUCT_SMITH_MAX_TURN_SECS — max seconds per turn +# CONSTRUCT_SMITH_ANTHROPIC_CACHE_CONTROL — on/off; public endpoint defaults on +# CONSTRUCT_SMITH_ANTHROPIC_BETAS — comma-separated anthropic-beta capabilities +# CONSTRUCT_SMITH_ANTHROPIC_CONTEXT_WINDOW_TOKENS — effective input window # CONSTRUCT_SMITH_HOOKS_JSON — inline JSON hooks config # CONSTRUCT_SMITH_HOOKS_CONFIG — path to a hooks config file # META_API_KEY / MODEL_API_KEY — Meta Model API credential diff --git a/docs/smith.md b/docs/smith.md index 8db8fc7c..7585571b 100644 --- a/docs/smith.md +++ b/docs/smith.md @@ -131,6 +131,10 @@ Each `[smith.models.]` entry sets: `api_key = "..."` inline (discouraged). If neither is set, the protocol's standard key env var is used (`OPENAI_API_KEY`, etc.). - `model` — default model name; override per call with `@:`. +- Anthropic profiles may also set `anthropic_cache_control = true|false`, + `anthropic_betas = ["..."]`, and + `anthropic_context_window_tokens = 1000000`. Cache control defaults on only + for the public Anthropic endpoint; compatible gateways must opt in. None of the direct-API-key providers needs a profile — the key plus the `:` prefix already reaches its public endpoint, and the same key @@ -160,6 +164,14 @@ model = "grok-4.6" provider = "meta" api_key_env = "META_API_KEY" model = "muse-spark-1.1" + +[smith.models.claude-long] +provider = "anthropic" +api_key_env = "ANTHROPIC_API_KEY" +model = "claude-sonnet-4-6" +anthropic_cache_control = true +anthropic_betas = ["context-1m-2025-08-07"] +anthropic_context_window_tokens = 1000000 ``` ```text @@ -281,17 +293,32 @@ window. Smith automatically manages context budget through two layers: -- **Auto-compaction**: When estimated tokens reach 65% of the model's effective - context window (`AUTO_COMPACT_RATIO = 0.65`), Smith requests a structured - summary of older history and prepends a `[Compacted earlier context]` turn, - preserving recent turn pairs verbatim (`DEFAULT_KEEP_PAIRS = 4`). You can also - trigger manual compaction anytime with `/compact [N]`. Auto-compaction is enabled - by default; disable with `CONSTRUCT_SMITH_AUTO_COMPACT=off` (or `0`/`false`). +- **Auto-compaction**: When the provider-anchored preflight count reaches 65% + of the model's effective context window (`AUTO_COMPACT_RATIO = 0.65`), Smith + requests a structured summary of older history and prepends a + `[Compacted earlier context]` turn, preserving recent turn pairs verbatim + (`DEFAULT_KEEP_PAIRS = 4`). You can also trigger manual compaction anytime + with `/compact [N]`. Auto-compaction is enabled by default; disable with + `CONSTRUCT_SMITH_AUTO_COMPACT=off` (or `0`/`false`). - **Rolling prune**: If context exceeds 70% utilization (`UTILIZATION = 0.70`), the oldest turn pairs are pruned, always keeping at least the two most-recent turn pairs. Smith learns and persists runtime limits when providers report overflow errors (spec 0070). +After each response, the provider's input-token count is authoritative. Before +the next request, Smith adds only a character-estimated delta for content added +or removed since that report. Before the first report (including after resume), +or when a provider omits usage, the whole request uses the character estimate +as a preflight fallback. Smith starts Muse Spark 1.1 with its advertised +one-million-token input window and retains the normal runtime limit-learning +fallback. + +First-party Anthropic API-key and Claude OAuth requests add prompt-cache +breakpoints to the stable system/tool prefix and recent user boundaries. Known +Claude Sonnet 4 models on the public API also enable Anthropic's one-million- +token context beta automatically; set the Anthropic capability options on a +named profile for a new model or compatible endpoint. + ### Ambient features that use smith Beyond running its own sessions, smith powers several daemon-level @@ -326,6 +353,12 @@ notice in the status bar that opens `/configure`. browser tools. - `CONSTRUCT_SMITH_AUTO_COMPACT=off` — disable auto-compaction before rolling prune. +- `CONSTRUCT_SMITH_ANTHROPIC_CACHE_CONTROL=on|off` — override prompt-cache + breakpoints for the built-in Anthropic API-key provider. +- `CONSTRUCT_SMITH_ANTHROPIC_BETAS=` — add comma-separated + `anthropic-beta` capabilities to the built-in Anthropic provider. +- `CONSTRUCT_SMITH_ANTHROPIC_CONTEXT_WINDOW_TOKENS=` — declare its + effective input window (for example `1000000`). - `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY` (or `GOOGLE_API_KEY`), `META_API_KEY` (or `MODEL_API_KEY`), `GROK_API_KEY` (or `XAI_API_KEY`), `DEEPSEEK_API_KEY`, `OPENROUTER_API_KEY` — API keys for each supported diff --git a/specs/0030-smith-model-profiles-are-named-endpoints.md b/specs/0030-smith-model-profiles-are-named-endpoints.md index 80b304c2..6e53b454 100644 --- a/specs/0030-smith-model-profiles-are-named-endpoints.md +++ b/specs/0030-smith-model-profiles-are-named-endpoints.md @@ -14,6 +14,12 @@ referenced with an explicit `@` prefix (optionally `@:` to override the model), usable anywhere a model spec is accepted — `--model`, `CONSTRUCT_SMITH_MODEL`, and the `/model` slash command. +Profiles may also carry protocol-scoped endpoint capabilities that cannot be +inferred from wire compatibility alone. Anthropic profiles can declare prompt +cache-control support, beta headers, and an effective context window. These +settings affect Smith's native provider only; routing may ignore capabilities +that its own translation layer does not implement. + Profiles exist so that multiple distinct endpoints — including several OpenAI-compatible vendors plus the first-party API — can coexist in a single session and be switched at runtime, which the single per-protocol base-URL env @@ -46,6 +52,9 @@ endpoint's URL and credential declared in one place. can differ between machines with different `config.toml` files. Resolution failures (missing profile, missing key, unknown provider) must report actionable errors rather than silently falling back to a different endpoint. +- Provider-specific capabilities must be opt-in for compatible third-party + endpoints. Sharing a wire dialect does not prove support for first-party + request fields or beta headers. ## Non-Goals @@ -69,6 +78,14 @@ provider = "openai" base_url = "https://api.groq.com/openai/v1" api_key_env = "GROQ_API_KEY" model = "llama-3.3-70b-versatile" + +[smith.models.claude-long] +provider = "anthropic" +api_key_env = "ANTHROPIC_API_KEY" +model = "claude-sonnet-4-6" +anthropic_cache_control = true +anthropic_betas = ["context-1m-2025-08-07"] +anthropic_context_window_tokens = 1000000 ``` In one session: `/model openai:gpt-5` reaches first-party OpenAI, then diff --git a/specs/0215-smith-provider-token-counts-are-authoritative.md b/specs/0215-smith-provider-token-counts-are-authoritative.md new file mode 100644 index 00000000..909d4c47 --- /dev/null +++ b/specs/0215-smith-provider-token-counts-are-authoritative.md @@ -0,0 +1,46 @@ +# 0215-smith-provider-token-counts-are-authoritative + +Status: accepted +Date: 2026-09-09 +Area: harness +Scope: Smith uses provider input-token usage as the source of truth for context pressure. + +## Decision + +After a successful model call, Smith treats the provider-reported total input +token count as authoritative for context-window state, compaction, pruning, and +limit-probe decisions. Before the next call, content added or removed since the +reported request is represented by a character-based delta estimate anchored +to that report. + +When a provider omits input usage, or a resumed session has not yet completed a +new model call, Smith may estimate the full pending request by characters. That +fallback is preflight-only and must not be presented as provider usage. A later +provider report replaces the fallback immediately. + +Providers whose APIs split prompt usage into fresh, cache-write, and cache-read +fields must report their sum as total input occupancy. A cached-token metric may +remain the cache-read subset, but it must not be subtracted from the context +total. + +## Reason + +Provider tokenizers account for message framing, tool schemas, cache prefixes, +and model-specific tokenization that a character ratio cannot reproduce. Using +the heuristic after a real count is available can compact too early or overrun +the actual window. New input still needs a preflight estimate because no +provider can report a request it has not received yet. + +## Consequences + +- Context mutations between calls are estimated relative to the latest real + count, rather than re-estimating the entire prompt. +- Missing usage remains an explicit state; zero must not silently mean unknown. +- Switching providers or resetting a conversation invalidates the prior anchor. +- Overflow errors and learned limits remain safety mechanisms, not substitutes + for successful-call usage. + +## Non-Goals + +This does not add a local provider tokenizer or persist an inferred token anchor +across process restarts. diff --git a/specs/0216-anthropic-prompt-cache-and-context-capabilities.md b/specs/0216-anthropic-prompt-cache-and-context-capabilities.md new file mode 100644 index 00000000..e50b999a --- /dev/null +++ b/specs/0216-anthropic-prompt-cache-and-context-capabilities.md @@ -0,0 +1,48 @@ +# 0216-anthropic-prompt-cache-and-context-capabilities + +Status: accepted +Date: 2026-09-09 +Area: harness +Scope: Smith declares Anthropic prompt-cache breakpoints and endpoint context capabilities without assuming every compatible endpoint is first-party. + +## Decision + +Requests to first-party Anthropic Messages endpoints place ephemeral cache +breakpoints on stable prompt prefixes: the system prompt, the tool-definition +prefix, and the newest user-message boundaries, up to the provider's four +breakpoint limit. First-party Claude OAuth requests use the same policy. +Anthropic-compatible third-party endpoints receive no cache directives by +default. + +For models known to require Anthropic's one-million-token context beta, the +first-party API-key provider sends the capability header and reports the larger +effective window to Smith's budget manager. Model profiles may explicitly +configure cache-control support, beta capabilities, and an effective context +window so new first-party models and compatible gateways can be represented +without changing cross-provider defaults. A conservative 200,000-token fallback +remains for Anthropic-compatible endpoints with no declared capability. + +## Reason + +Smith resends a large, mostly stable system/tool prefix on every tool step. +Correct cache boundaries make that prefix reusable, while the larger-context +header must agree with the budget Smith enforces. Anthropic wire compatibility +alone does not imply support for Anthropic-specific cache fields or betas, so +blindly enabling them can break gateways and other providers. + +## Consequences + +- Cache directives stay inside first-party Anthropic request construction unless + a profile opts a compatible endpoint in. +- For a beta-gated model, effective-window reporting and beta headers must + agree; models/endpoints where the larger window is stable may declare the + window without an obsolete beta. +- Cache-write and cache-read usage contributes to total input-window occupancy. +- Capability configuration is additive and provider-scoped, so unrelated Smith + providers retain their existing request shapes. + +## Non-Goals + +This does not promise that every account is entitled to every Anthropic beta, +and it does not inject Anthropic cache fields into other Messages-compatible +services.