From 9c7f8ea76dbd04df744c2c4f34e23b58c3bd63a8 Mon Sep 17 00:00:00 2001 From: Edwin Date: Wed, 9 Sep 2026 22:05:51 -0700 Subject: [PATCH] feat(smith): accept native image input --- crates/adapter-smith/src/agent.rs | 25 +- crates/adapter-smith/src/compact.rs | 10 + crates/adapter-smith/src/context.rs | 8 + crates/adapter-smith/src/image_input.rs | 356 ++++++++++++++++++ crates/adapter-smith/src/interactive.rs | 26 +- crates/adapter-smith/src/lib.rs | 1 + crates/adapter-smith/src/persist.rs | 28 +- .../adapter-smith/src/provider/anthropic.rs | 47 +++ .../src/provider/antigravity_oauth.rs | 19 + .../src/provider/claude_oauth.rs | 4 + .../adapter-smith/src/provider/codex_oauth.rs | 47 +++ crates/adapter-smith/src/provider/gemini.rs | 41 ++ .../adapter-smith/src/provider/kimi_oauth.rs | 4 + crates/adapter-smith/src/provider/meta.rs | 47 +++ crates/adapter-smith/src/provider/mod.rs | 57 +++ crates/adapter-smith/src/provider/ollama.rs | 32 ++ crates/adapter-smith/src/provider/openai.rs | 92 ++++- crates/adapter-smith/src/provider_watchdog.rs | 1 + docs/smith.md | 22 ++ specs/0142-smith-meta-model-api.md | 2 +- ...14-smith-image-input-is-provider-native.md | 62 +++ 21 files changed, 922 insertions(+), 9 deletions(-) create mode 100644 crates/adapter-smith/src/image_input.rs create mode 100644 specs/0214-smith-image-input-is-provider-native.md diff --git a/crates/adapter-smith/src/agent.rs b/crates/adapter-smith/src/agent.rs index ebda9bc1..c6e054a5 100644 --- a/crates/adapter-smith/src/agent.rs +++ b/crates/adapter-smith/src/agent.rs @@ -763,6 +763,25 @@ pub async fn run( if user_text.trim().is_empty() { continue; } + let user_content = match crate::image_input::user_content(user_text.clone(), &cwd) { + Ok(content) => content, + Err(e) => { + emit.emit(SessionEvent::Error { + message: format!("image input error: {e:#}"), + }); + continue; + } + }; + if let Err(e) = provider::ensure_image_input_supported( + provider.as_ref(), + &messages, + Some(&user_content), + ) { + emit.emit(SessionEvent::Error { + message: e.to_string(), + }); + continue; + } hooks .run( "user_prompt_submit", @@ -779,7 +798,7 @@ pub async fn run( persist, Message { role: Role::User, - content: Content::Text { text: user_text }, + content: user_content, } ); @@ -1723,7 +1742,7 @@ pub fn resolve_model_from_spec(spec_str: &str) -> Result { Some(GROK_BASE_URL.to_string()), grok_api_key()?, )?), - provider::routing::Provider::DeepSeek => Box::new(provider::openai::OpenAi::with_config( + provider::routing::Provider::DeepSeek => Box::new(provider::openai::OpenAi::text_only( Some(DEEPSEEK_BASE_URL.to_string()), deepseek_api_key()?, )?), @@ -1839,7 +1858,7 @@ fn build_profile_model( base_url.or_else(|| Some(GROK_BASE_URL.to_string())), profile_api_key(profile, name, &["GROK_API_KEY", "XAI_API_KEY"])?, )?), - provider::routing::Provider::DeepSeek => Box::new(provider::openai::OpenAi::with_config( + provider::routing::Provider::DeepSeek => Box::new(provider::openai::OpenAi::text_only( base_url.or_else(|| Some(DEEPSEEK_BASE_URL.to_string())), profile_api_key(profile, name, &["DEEPSEEK_API_KEY"])?, )?), diff --git a/crates/adapter-smith/src/compact.rs b/crates/adapter-smith/src/compact.rs index 5243685e..436a1d68 100644 --- a/crates/adapter-smith/src/compact.rs +++ b/crates/adapter-smith/src/compact.rs @@ -251,6 +251,11 @@ fn render_head_for_summarizer(head: &[Message]) -> String { out.push_str(text.trim()); out.push_str("\n\n"); } + (Role::User, Content::UserInput { text, images }) => { + out.push_str("USER: "); + out.push_str(text.trim()); + out.push_str(&format!("\n[{} image attachment(s)]\n\n", images.len())); + } (_, Content::AssistantToolCalls { text, calls }) => { if let Some(t) = text { if !t.is_empty() { @@ -304,6 +309,11 @@ fn render_head_for_summarizer(head: &[Message]) -> String { out.push_str(text.trim()); out.push_str("\n\n"); } + (Role::System | Role::Assistant | Role::Tool, Content::UserInput { text, images }) => { + out.push_str("OTHER: "); + out.push_str(text.trim()); + out.push_str(&format!("\n[{} image attachment(s)]\n\n", images.len())); + } } } out diff --git a/crates/adapter-smith/src/context.rs b/crates/adapter-smith/src/context.rs index 0237b4da..fb5de325 100644 --- a/crates/adapter-smith/src/context.rs +++ b/crates/adapter-smith/src/context.rs @@ -186,6 +186,14 @@ pub fn estimate_tokens(messages: &[Message]) -> usize { for m in messages { match &m.content { Content::Text { text: t } => chars += t.len(), + Content::UserInput { text, images } => { + chars += text.len(); + // Image tokenization is provider- and resolution-dependent. + // A conservative fixed allowance keeps base64 storage bytes + // from masquerading as text tokens while still budgeting the + // visual context carried by each image. + chars += images.len() * 6_000; + } Content::AssistantToolCalls { text, calls } => { if let Some(t) = text { chars += t.len(); diff --git a/crates/adapter-smith/src/image_input.rs b/crates/adapter-smith/src/image_input.rs new file mode 100644 index 00000000..695fbd85 --- /dev/null +++ b/crates/adapter-smith/src/image_input.rs @@ -0,0 +1,356 @@ +//! Turn explicit image paths in a user prompt into stable, provider-agnostic +//! image inputs. +//! +//! Construct's clipboard and drag/drop paths write binary payloads into the +//! session attachment directory, then paste the daemon-host path into the +//! harness. Smith also accepts paths supplied directly by users (including +//! relative paths and the `[#file:…]` attachment token used by web clients). +//! We snapshot bytes at submit time so replay after a daemon restart sends the +//! same image even if the original project file later changes. + +use crate::provider::{Content, ImageInput}; +use anyhow::{bail, Context, Result}; +use base64::Engine as _; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +/// Anthropic's per-image limit is the tightest of Smith's native providers. +/// Applying it at the shared boundary keeps a persisted turn portable across +/// later `/model` switches rather than accepting a payload another provider +/// cannot replay. +const MAX_IMAGE_BYTES: u64 = 5 * 1024 * 1024; +const MAX_IMAGES_PER_TURN: usize = 20; + +/// Preserve text-only turns byte-for-byte. A multimodal turn retains the same +/// text and adds snapshots for every explicit local image reference found. +pub fn user_content(text: String, cwd: &Path) -> Result { + let candidates = image_path_candidates(&text, cwd); + if candidates.is_empty() { + return Ok(Content::Text { text }); + } + if candidates.len() > MAX_IMAGES_PER_TURN { + bail!( + "image input contains {} images (maximum {})", + candidates.len(), + MAX_IMAGES_PER_TURN + ); + } + + let images = candidates + .into_iter() + .map(|path| load_image(&path)) + .collect::>>()?; + Ok(Content::UserInput { text, images }) +} + +fn load_image(path: &Path) -> Result { + let metadata = std::fs::metadata(path) + .with_context(|| format!("image input not found: {}", path.display()))?; + if !metadata.is_file() { + bail!("image input is not a file: {}", path.display()); + } + if metadata.len() == 0 { + bail!("image input is empty: {}", path.display()); + } + if metadata.len() > MAX_IMAGE_BYTES { + bail!( + "image input is too large: {} is {} bytes (maximum {} bytes)", + path.display(), + metadata.len(), + MAX_IMAGE_BYTES + ); + } + + let bytes = + std::fs::read(path).with_context(|| format!("read image input {}", path.display()))?; + if bytes.len() as u64 > MAX_IMAGE_BYTES { + bail!( + "image input is too large: {} is {} bytes (maximum {} bytes)", + path.display(), + bytes.len(), + MAX_IMAGE_BYTES + ); + } + let format = image::guess_format(&bytes) + .with_context(|| format!("unsupported or invalid image input: {}", path.display()))?; + let media_type = match format { + image::ImageFormat::Png => "image/png", + image::ImageFormat::Jpeg => "image/jpeg", + image::ImageFormat::Gif => "image/gif", + image::ImageFormat::WebP => "image/webp", + other => bail!( + "unsupported image format {:?} for {}; use PNG, JPEG, GIF, or WebP", + other, + path.display() + ), + }; + + Ok(ImageInput { + media_type: media_type.to_string(), + data: base64::engine::general_purpose::STANDARD.encode(bytes), + source: Some(path.display().to_string()), + }) +} + +fn image_path_candidates(text: &str, cwd: &Path) -> Vec { + let mut raw = Vec::new(); + collect_file_references(text, &mut raw); + collect_markdown_images(text, &mut raw); + + let trimmed = text.trim(); + if looks_like_standalone_path(trimmed) { + raw.push(trimmed.to_string()); + } else { + for word in terminal_tokens(text) { + let token = word.trim_matches(|c: char| { + matches!( + c, + '"' | '\'' | '`' | ',' | ';' | ':' | '(' | ')' | '[' | ']' + ) + }); + if looks_like_path(token) { + raw.push(token.to_string()); + } + } + } + + let mut seen = HashSet::new(); + let mut paths = Vec::new(); + for candidate in raw { + let Some(path) = normalize_candidate(&candidate, cwd) else { + continue; + }; + if !has_image_extension(&path) { + continue; + } + let key = path.to_string_lossy().into_owned(); + if seen.insert(key) { + paths.push(path); + } + } + paths +} + +fn collect_file_references(text: &str, out: &mut Vec) { + let mut rest = text; + while let Some(start) = rest.find("[#file:") { + let value = &rest[start + "[#file:".len()..]; + let Some(end) = value.find(']') else { + break; + }; + out.push(value[..end].to_string()); + rest = &value[end + 1..]; + } +} + +fn collect_markdown_images(text: &str, out: &mut Vec) { + let mut rest = text; + while let Some(start) = rest.find("![") { + let after_alt = &rest[start + 2..]; + let Some(close_alt) = after_alt.find("](") else { + break; + }; + let target = &after_alt[close_alt + 2..]; + let Some(close_target) = target.find(')') else { + break; + }; + out.push(target[..close_target].to_string()); + rest = &target[close_target + 1..]; + } +} + +fn looks_like_path(raw: &str) -> bool { + let token = raw + .trim() + .trim_matches(|c| matches!(c, '"' | '\'' | '`' | '<' | '>')); + if token.is_empty() || token.contains('\n') { + return false; + } + if token.contains("://") && !token.starts_with("file://") { + return false; + } + let pathish = token.starts_with('/') + || token.starts_with("./") + || token.starts_with("../") + || token.starts_with("~/") + || token.starts_with("file://"); + pathish && has_image_extension(Path::new(token)) +} + +fn looks_like_standalone_path(raw: &str) -> bool { + let quoted = (raw.starts_with('"') && raw.ends_with('"')) + || (raw.starts_with('\'') && raw.ends_with('\'')); + let token = raw + .trim() + .trim_matches(|c| matches!(c, '"' | '\'' | '`' | '<' | '>')); + if token.contains("://") && !token.starts_with("file://") { + return false; + } + looks_like_path(token) + || (quoted && has_image_extension(Path::new(token))) + || (!token.chars().any(char::is_whitespace) && has_image_extension(Path::new(token))) + || (token.contains("\\ ") && has_image_extension(Path::new(token))) +} + +/// Split ordinary prose while retaining terminal-escaped spaces in a pasted +/// path. Quotes are retained for the normal trimming pass above. +fn terminal_tokens(text: &str) -> Vec { + let mut tokens = Vec::new(); + let mut current = String::new(); + let mut chars = text.chars().peekable(); + let mut quote = None; + while let Some(c) = chars.next() { + if c == '\\' { + current.push(c); + if let Some(next) = chars.next() { + current.push(next); + } + } else if matches!(c, '"' | '\'') { + current.push(c); + if quote == Some(c) { + quote = None; + } else if quote.is_none() { + quote = Some(c); + } + } else if c.is_whitespace() && quote.is_none() { + if !current.is_empty() { + tokens.push(std::mem::take(&mut current)); + } + } else { + current.push(c); + } + } + if !current.is_empty() { + tokens.push(current); + } + tokens +} + +fn normalize_candidate(raw: &str, cwd: &Path) -> Option { + let mut token = raw + .trim() + .trim_matches(|c| matches!(c, '"' | '\'' | '`' | '<' | '>')) + .to_string(); + if token.contains("://") && !token.starts_with("file://") { + return None; + } + if let Some(path) = token.strip_prefix("file://") { + token = path.to_string(); + } + token = unescape_terminal_path(&token)?; + + if let Some(rest) = token.strip_prefix("~/") { + let home = std::env::var_os("HOME")?; + return Some(PathBuf::from(home).join(rest)); + } + let path = PathBuf::from(token); + Some(if path.is_absolute() { + path + } else { + cwd.join(path) + }) +} + +fn unescape_terminal_path(raw: &str) -> Option { + let mut out = String::with_capacity(raw.len()); + let mut chars = raw.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + out.push(chars.next()?); + } else { + out.push(c); + } + } + Some(out) +} + +fn has_image_extension(path: &Path) -> bool { + path.extension() + .and_then(|ext| ext.to_str()) + .map(|ext| ext.to_ascii_lowercase()) + .is_some_and(|ext| { + matches!( + ext.as_str(), + "png" | "jpg" | "jpeg" | "gif" | "webp" | "bmp" | "tif" | "tiff" | "heic" + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn write_png(path: &Path) { + let image = image::RgbaImage::from_pixel(2, 1, image::Rgba([1, 2, 3, 255])); + let mut bytes = Cursor::new(Vec::new()); + image::DynamicImage::ImageRgba8(image) + .write_to(&mut bytes, image::ImageFormat::Png) + .unwrap(); + std::fs::write(path, bytes.into_inner()).unwrap(); + } + + #[test] + fn text_without_an_explicit_image_stays_plain_text() { + let content = + user_content("please edit image.png later".into(), Path::new("/tmp")).unwrap(); + assert!(matches!(content, Content::Text { text } if text == "please edit image.png later")); + + let content = + user_content("please edit the screenshot later".into(), Path::new("/tmp")).unwrap(); + assert!( + matches!(content, Content::Text { text } if text == "please edit the screenshot later") + ); + + let remote = "![remote](https://example.com/image.png)"; + let content = user_content(remote.into(), Path::new("/tmp")).unwrap(); + assert!(matches!(content, Content::Text { text } if text == remote)); + } + + #[test] + fn missing_standalone_image_is_reported() { + let error = user_content("missing.png".into(), Path::new("/tmp")).unwrap_err(); + assert!(error.to_string().contains("image input not found")); + } + + #[test] + fn loads_bare_attachment_and_file_reference_paths() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("shot.png"); + write_png(&path); + + for prompt in [ + path.display().to_string(), + format!("inspect [#file:{}]", path.display()), + "inspect ./shot.png".to_string(), + "inspect ![shot](./shot.png)".to_string(), + ] { + let content = user_content(prompt.clone(), dir.path()).unwrap(); + let Content::UserInput { text, images } = content else { + panic!("expected image input for {prompt}"); + }; + assert_eq!(text, prompt); + assert_eq!(images.len(), 1); + assert_eq!(images[0].media_type, "image/png"); + assert!(!images[0].data.is_empty()); + } + } + + #[test] + fn loads_terminal_escaped_space_and_deduplicates_references() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("screen shot.png"); + write_png(&path); + let escaped = path.display().to_string().replace(' ', "\\ "); + let prompt = format!("{escaped} [#file:{}]", path.display()); + let content = user_content(prompt, dir.path()).unwrap(); + let Content::UserInput { images, .. } = content else { + panic!("expected image input"); + }; + assert_eq!(images.len(), 1); + + let quoted = format!("inspect \"{}\"", path.display()); + let content = user_content(quoted, dir.path()).unwrap(); + assert!(matches!(content, Content::UserInput { images, .. } if images.len() == 1)); + } +} diff --git a/crates/adapter-smith/src/interactive.rs b/crates/adapter-smith/src/interactive.rs index 187a8cca..ec62fa5c 100644 --- a/crates/adapter-smith/src/interactive.rs +++ b/crates/adapter-smith/src/interactive.rs @@ -2592,6 +2592,28 @@ pub async fn run( } } + let user_content = match crate::image_input::user_content(user_text.clone(), &cwd) { + Ok(content) => content, + Err(e) => { + let message = format!("image input error: {e:#}"); + term.note(&format!("({message})")); + emit.emit(SessionEvent::Error { message }); + emit_editor_state(&emit, &editor, &queue); + continue; + } + }; + if let Err(e) = provider::ensure_image_input_supported( + provider.as_ref(), + &messages, + Some(&user_content), + ) { + let message = e.to_string(); + term.note(&format!("({message})")); + emit.emit(SessionEvent::Error { message }); + emit_editor_state(&emit, &editor, &queue); + continue; + } + hooks .run( "user_prompt_submit", @@ -2609,9 +2631,7 @@ pub async fn run( persist, Message { role: Role::User, - content: Content::Text { - text: user_text.clone() - }, + content: user_content, } ); // Echo an OBSERVATION trigger into the panel (dim) before the response diff --git a/crates/adapter-smith/src/lib.rs b/crates/adapter-smith/src/lib.rs index 1b2853de..7189bf36 100644 --- a/crates/adapter-smith/src/lib.rs +++ b/crates/adapter-smith/src/lib.rs @@ -9,6 +9,7 @@ mod agent; mod compact; mod context; mod hooks; +mod image_input; mod interactive; mod interval_suggest; mod model_limits; diff --git a/crates/adapter-smith/src/persist.rs b/crates/adapter-smith/src/persist.rs index b674ee8f..a6dbdc47 100644 --- a/crates/adapter-smith/src/persist.rs +++ b/crates/adapter-smith/src/persist.rs @@ -173,7 +173,33 @@ pub fn is_resume() -> bool { #[cfg(test)] mod tests { use super::*; - use crate::provider::{Content, Message, Role}; + use crate::provider::{Content, ImageInput, Message, Role}; + + #[test] + fn image_turn_round_trips_without_rereading_its_source() { + let dir = tempfile::tempdir().unwrap(); + let mut persist = Persist::open(Some(dir.path())).unwrap(); + persist.append(&Message { + role: Role::User, + content: Content::UserInput { + text: "/tmp/now-gone.png".into(), + images: vec![ImageInput { + media_type: "image/png".into(), + data: "c25hcHNob3Q=".into(), + source: Some("/tmp/now-gone.png".into()), + }], + }, + }); + + let loaded = Persist::load(persist.path()).unwrap(); + let Content::UserInput { text, images } = &loaded[0].content else { + panic!("expected persisted image turn"); + }; + assert_eq!(text, "/tmp/now-gone.png"); + assert_eq!(images[0].media_type, "image/png"); + assert_eq!(images[0].data, "c25hcHNob3Q="); + assert_eq!(images[0].source.as_deref(), Some("/tmp/now-gone.png")); + } #[test] fn reset_truncates_persisted_messages_and_keeps_appending() { diff --git a/crates/adapter-smith/src/provider/anthropic.rs b/crates/adapter-smith/src/provider/anthropic.rs index f61588ef..e257237b 100644 --- a/crates/adapter-smith/src/provider/anthropic.rs +++ b/crates/adapter-smith/src/provider/anthropic.rs @@ -65,6 +65,23 @@ pub(crate) fn messages_to_anthropic(messages: &[Message]) -> Vec { }; out.push(json!({ "role": role, "content": text })); } + (_, Content::UserInput { text, images }) => { + let mut blocks = Vec::with_capacity(images.len() + 1); + if !text.is_empty() { + blocks.push(json!({ "type": "text", "text": text })); + } + blocks.extend(images.iter().map(|image| { + json!({ + "type": "image", + "source": { + "type": "base64", + "media_type": image.media_type, + "data": image.data, + } + }) + })); + out.push(json!({ "role": "user", "content": blocks })); + } (_, Content::AssistantToolCalls { text, calls }) => { let mut blocks: Vec = Vec::with_capacity(calls.len() + 1); if let Some(t) = text { @@ -303,6 +320,10 @@ impl LlmProvider for Anthropic { "anthropic" } + fn supports_image_input(&self) -> bool { + true + } + async fn complete( &self, model: &str, @@ -358,3 +379,29 @@ enum BlockKind { /// these to `TextSink::reasoning_delta` instead of `delta`. Thinking, } + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider::ImageInput; + + #[test] + fn user_images_use_anthropic_source_blocks() { + let wire = messages_to_anthropic(&[Message { + role: Role::User, + content: Content::UserInput { + text: "inspect".into(), + images: vec![ImageInput { + media_type: "image/jpeg".into(), + data: "YWJj".into(), + source: None, + }], + }, + }]); + assert_eq!(wire[0]["content"][0]["type"], "text"); + assert_eq!(wire[0]["content"][1]["type"], "image"); + assert_eq!(wire[0]["content"][1]["source"]["type"], "base64"); + assert_eq!(wire[0]["content"][1]["source"]["media_type"], "image/jpeg"); + assert_eq!(wire[0]["content"][1]["source"]["data"], "YWJj"); + } +} diff --git a/crates/adapter-smith/src/provider/antigravity_oauth.rs b/crates/adapter-smith/src/provider/antigravity_oauth.rs index 4f3a2298..3d10b257 100644 --- a/crates/adapter-smith/src/provider/antigravity_oauth.rs +++ b/crates/adapter-smith/src/provider/antigravity_oauth.rs @@ -156,6 +156,10 @@ impl LlmProvider for AntigravityOauth { "antigravity-oauth" } + fn supports_image_input(&self) -> bool { + true + } + async fn complete( &self, model: &str, @@ -298,6 +302,21 @@ fn messages_to_antigravity(messages: &[Message]) -> Vec { }; out.push(json!({ "role": role, "parts": [{ "text": text }] })); } + (_, Content::UserInput { text, images }) => { + let mut parts = Vec::with_capacity(images.len() + 1); + if !text.is_empty() { + parts.push(json!({ "text": text })); + } + parts.extend(images.iter().map(|image| { + json!({ + "inlineData": { + "mimeType": image.media_type, + "data": image.data, + } + }) + })); + out.push(json!({ "role": "user", "parts": parts })); + } (_, Content::AssistantToolCalls { text, calls }) => { let mut parts: Vec = Vec::new(); if let Some(t) = text { diff --git a/crates/adapter-smith/src/provider/claude_oauth.rs b/crates/adapter-smith/src/provider/claude_oauth.rs index 16e83e9f..85ed27e8 100644 --- a/crates/adapter-smith/src/provider/claude_oauth.rs +++ b/crates/adapter-smith/src/provider/claude_oauth.rs @@ -344,6 +344,10 @@ impl LlmProvider for ClaudeOauth { "claude-oauth" } + fn supports_image_input(&self) -> bool { + true + } + async fn complete( &self, model: &str, diff --git a/crates/adapter-smith/src/provider/codex_oauth.rs b/crates/adapter-smith/src/provider/codex_oauth.rs index e5a4f005..9f1e1dc4 100644 --- a/crates/adapter-smith/src/provider/codex_oauth.rs +++ b/crates/adapter-smith/src/provider/codex_oauth.rs @@ -741,6 +741,22 @@ fn message_to_input_items(m: &Message) -> Vec { "content": [{ "type": typ, "text": text }], })] } + Content::UserInput { text, images } => { + let mut content = Vec::with_capacity(images.len() + 1); + if !text.is_empty() { + content.push(json!({ "type": "input_text", "text": text })); + } + content.extend( + images + .iter() + .map(|image| json!({ "type": "input_image", "image_url": image.data_url() })), + ); + vec![json!({ + "type": "message", + "role": "user", + "content": content, + })] + } Content::AssistantToolCalls { text, calls } => { let mut out = Vec::with_capacity(calls.len() + 1); if let Some(t) = text.as_deref().filter(|t| !t.is_empty()) { @@ -893,6 +909,10 @@ impl LlmProvider for CodexOauth { "codex-oauth" } + fn supports_image_input(&self) -> bool { + true + } + async fn complete( &self, model: &str, @@ -1579,6 +1599,33 @@ mod tests { assert!(body.get("parallel_tool_calls").is_none()); } + #[test] + fn build_body_emits_responses_image_shape() { + let body = build_responses_body( + "gpt-5-codex", + "system", + &[Message { + role: Role::User, + content: Content::UserInput { + text: "inspect".into(), + images: vec![crate::provider::ImageInput { + media_type: "image/png".into(), + data: "YWJj".into(), + source: Some("/tmp/not-sent.png".into()), + }], + }, + }], + &[], + ); + assert_eq!(body["input"][0]["content"][0]["type"], "input_text"); + assert_eq!(body["input"][0]["content"][1]["type"], "input_image"); + assert_eq!( + body["input"][0]["content"][1]["image_url"], + "data:image/png;base64,YWJj" + ); + assert!(!body.to_string().contains("not-sent.png")); + } + /// Construct injects one stable session id into the Smith adapter /// environment. Codex OAuth must copy it into every Responses request so /// successive turns route to the same prompt-cache node. diff --git a/crates/adapter-smith/src/provider/gemini.rs b/crates/adapter-smith/src/provider/gemini.rs index 70a43654..0bc7dbe6 100644 --- a/crates/adapter-smith/src/provider/gemini.rs +++ b/crates/adapter-smith/src/provider/gemini.rs @@ -74,6 +74,21 @@ fn messages_to_gemini(messages: &[Message]) -> Vec { }; out.push(json!({ "role": role, "parts": [{ "text": text }] })); } + (_, Content::UserInput { text, images }) => { + let mut parts = Vec::with_capacity(images.len() + 1); + if !text.is_empty() { + parts.push(json!({ "text": text })); + } + parts.extend(images.iter().map(|image| { + json!({ + "inlineData": { + "mimeType": image.media_type, + "data": image.data, + } + }) + })); + out.push(json!({ "role": "user", "parts": parts })); + } (_, Content::AssistantToolCalls { text, calls }) => { let mut parts: Vec = Vec::with_capacity(calls.len() + 1); if let Some(t) = text { @@ -149,6 +164,10 @@ impl LlmProvider for Gemini { "gemini" } + fn supports_image_input(&self) -> bool { + true + } + async fn complete( &self, model: &str, @@ -305,6 +324,28 @@ impl LlmProvider for Gemini { #[cfg(test)] mod tests { use super::*; + use crate::provider::ImageInput; + + #[test] + fn maps_user_images_to_inline_data_parts() { + let contents = messages_to_gemini(&[Message { + role: Role::User, + content: Content::UserInput { + text: "inspect".into(), + images: vec![ImageInput { + media_type: "image/webp".into(), + data: "YWJj".into(), + source: None, + }], + }, + }]); + assert_eq!(contents[0]["parts"][0]["text"], "inspect"); + assert_eq!( + contents[0]["parts"][1]["inlineData"]["mimeType"], + "image/webp" + ); + assert_eq!(contents[0]["parts"][1]["inlineData"]["data"], "YWJj"); + } #[test] fn maps_roles_and_tool_round_trip() { diff --git a/crates/adapter-smith/src/provider/kimi_oauth.rs b/crates/adapter-smith/src/provider/kimi_oauth.rs index e1781cc8..693f9e8a 100644 --- a/crates/adapter-smith/src/provider/kimi_oauth.rs +++ b/crates/adapter-smith/src/provider/kimi_oauth.rs @@ -229,6 +229,10 @@ impl LlmProvider for KimiOauth { "kimi-oauth" } + fn supports_image_input(&self) -> bool { + true + } + async fn complete( &self, model: &str, diff --git a/crates/adapter-smith/src/provider/meta.rs b/crates/adapter-smith/src/provider/meta.rs index 7361b34c..f4e529a9 100644 --- a/crates/adapter-smith/src/provider/meta.rs +++ b/crates/adapter-smith/src/provider/meta.rs @@ -60,6 +60,22 @@ fn message_to_input_items(message: &Message) -> Vec { "content": [{ "type": kind, "text": text }], })] } + Content::UserInput { text, images } => { + let mut content = Vec::with_capacity(images.len() + 1); + if !text.is_empty() { + content.push(json!({ "type": "input_text", "text": text })); + } + content.extend( + images + .iter() + .map(|image| json!({ "type": "input_image", "image_url": image.data_url() })), + ); + vec![json!({ + "type": "message", + "role": "user", + "content": content, + })] + } Content::AssistantToolCalls { text, calls } => { let mut items = Vec::with_capacity(calls.len() + 1); if let Some(text) = text.as_deref().filter(|text| !text.is_empty()) { @@ -146,6 +162,10 @@ impl LlmProvider for Meta { "meta" } + fn supports_image_input(&self) -> bool { + true + } + async fn complete( &self, model: &str, @@ -335,6 +355,33 @@ impl LlmProvider for Meta { #[cfg(test)] mod tests { use super::*; + use crate::provider::ImageInput; + + #[test] + fn request_uses_responses_image_input_shape() { + let body = build_body( + "muse-spark-1.1", + "", + &[Message { + role: Role::User, + content: Content::UserInput { + text: "inspect".into(), + images: vec![ImageInput { + media_type: "image/png".into(), + data: "YWJj".into(), + source: None, + }], + }, + }], + &[], + ); + assert_eq!(body["input"][0]["content"][0]["type"], "input_text"); + assert_eq!(body["input"][0]["content"][1]["type"], "input_image"); + assert_eq!( + body["input"][0]["content"][1]["image_url"], + "data:image/png;base64,YWJj" + ); + } #[test] fn request_uses_responses_message_and_tool_shapes() { diff --git a/crates/adapter-smith/src/provider/mod.rs b/crates/adapter-smith/src/provider/mod.rs index ac038368..03b03ecb 100644 --- a/crates/adapter-smith/src/provider/mod.rs +++ b/crates/adapter-smith/src/provider/mod.rs @@ -39,11 +39,35 @@ pub struct Message { pub content: Content, } +/// One image snapshot attached to a user turn. `data` is unwrapped base64 so +/// every provider can place it in its native content-block shape. `source` is +/// retained for diagnostics/transcript persistence but is never sent as image +/// bytes by itself, so replay does not depend on the source file still existing. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ImageInput { + pub media_type: String, + pub data: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, +} + +impl ImageInput { + pub fn data_url(&self) -> String { + format!("data:{};base64,{}", self.media_type, self.data) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum Content { /// Plain text (system / user / assistant). Text { text: String }, + /// A user turn with one or more local image snapshots. The original text + /// remains intact for transcript display and provider context. + UserInput { + text: String, + images: Vec, + }, /// Assistant turn that's making tool calls. May also include final /// pre-tool prose (`text`) that comes before the calls. AssistantToolCalls { @@ -78,6 +102,32 @@ pub enum Content { Reasoning(ReasoningItem), } +impl Content { + pub fn has_images(&self) -> bool { + matches!(self, Self::UserInput { images, .. } if !images.is_empty()) + } +} + +/// Reject multimodal content before any provider request when the selected +/// wire cannot represent it. Validation covers both the pending user turn and +/// existing history, including a resumed/image-bearing conversation switched +/// to a text-only provider. +pub fn ensure_image_input_supported( + provider: &dyn LlmProvider, + messages: &[Message], + pending: Option<&Content>, +) -> Result<()> { + let has_images = + pending.is_some_and(Content::has_images) || messages.iter().any(|m| m.content.has_images()); + if provider.supports_image_input() || !has_images { + return Ok(()); + } + anyhow::bail!( + "{} does not support image input; switch to an image-capable Smith provider or use /reset before continuing", + provider.name() + ) +} + /// A reasoning item captured from a Responses-API turn and replayed on the /// next request. `encrypted_content` is the opaque blob the backend returns /// when `include: ["reasoning.encrypted_content"]` is requested. @@ -460,6 +510,13 @@ mod overflow_tests { pub trait LlmProvider: Send + Sync { fn name(&self) -> &str; + /// Whether this wire implementation can represent image content. A model + /// behind a capable wire may still reject vision; that provider response + /// remains the authoritative model-level error. + fn supports_image_input(&self) -> bool { + false + } + /// Return the context window the provider will actually use for this /// model, when it exposes that runtime value. Providers whose effective /// allocation cannot be discovered cheaply leave this unknown and let diff --git a/crates/adapter-smith/src/provider/ollama.rs b/crates/adapter-smith/src/provider/ollama.rs index d1c9cfaf..52768016 100644 --- a/crates/adapter-smith/src/provider/ollama.rs +++ b/crates/adapter-smith/src/provider/ollama.rs @@ -126,6 +126,13 @@ fn messages_to_ollama(system: &str, messages: &[Message]) -> Vec { Content::Text { text } => { out.push(json!({ "role": role_str(m.role), "content": text })); } + Content::UserInput { text, images } => { + out.push(json!({ + "role": "user", + "content": text, + "images": images.iter().map(|image| image.data.as_str()).collect::>(), + })); + } Content::AssistantToolCalls { text, calls } => { let tc: Vec = calls .iter() @@ -190,6 +197,10 @@ impl LlmProvider for Ollama { "ollama" } + fn supports_image_input(&self) -> bool { + true + } + async fn effective_context_window_tokens(&self, model: &str) -> Option { if let Some(window) = self.loaded_context_window_tokens(model).await { return Some(window); @@ -328,6 +339,27 @@ impl LlmProvider for Ollama { #[cfg(test)] mod tests { use super::*; + use crate::provider::ImageInput; + + #[test] + fn maps_user_images_to_ollama_images_array() { + let messages = messages_to_ollama( + "", + &[Message { + role: Role::User, + content: Content::UserInput { + text: "inspect".into(), + images: vec![ImageInput { + media_type: "image/png".into(), + data: "YWJj".into(), + source: None, + }], + }, + }], + ); + assert_eq!(messages[0]["content"], "inspect"); + assert_eq!(messages[0]["images"], json!(["YWJj"])); + } #[test] fn ps_context_matches_implicit_latest_name() { diff --git a/crates/adapter-smith/src/provider/openai.rs b/crates/adapter-smith/src/provider/openai.rs index 65114961..bca3773e 100644 --- a/crates/adapter-smith/src/provider/openai.rs +++ b/crates/adapter-smith/src/provider/openai.rs @@ -23,6 +23,8 @@ pub struct OpenAi { /// (the response's final usage chunk then carries the generation's real /// USD cost) and the attribution headers its ranking honors. openrouter: bool, + image_input: bool, + provider_name: &'static str, } impl OpenAi { @@ -47,9 +49,21 @@ impl OpenAi { base_url, api_key, openrouter: false, + image_input: true, + provider_name: "openai", }) } + /// DeepSeek's chat-completions endpoint is text-only. Keep using the + /// shared dialect implementation while advertising that limitation at + /// Smith's input boundary. + pub fn text_only(base_url: Option, api_key: String) -> Result { + let mut provider = Self::with_config(base_url, api_key)?; + provider.image_input = false; + provider.provider_name = "deepseek"; + Ok(provider) + } + /// Build for an OpenRouter endpoint: same wire format, plus the /// OpenRouter-only request extras (usage-cost accounting, attribution /// headers). Kept a flag rather than a separate provider impl because @@ -80,6 +94,19 @@ fn messages_to_openai(system: &str, messages: &[Message]) -> Vec { Content::Text { text } => { out.push(json!({ "role": role_str(m.role), "content": text })); } + Content::UserInput { text, images } => { + let mut blocks = Vec::with_capacity(images.len() + 1); + if !text.is_empty() { + blocks.push(json!({ "type": "text", "text": text })); + } + blocks.extend(images.iter().map(|image| { + json!({ + "type": "image_url", + "image_url": { "url": image.data_url() }, + }) + })); + out.push(json!({ "role": "user", "content": blocks })); + } Content::AssistantToolCalls { text, calls } => { let tool_calls: Vec = calls .iter() @@ -146,7 +173,11 @@ fn tools_to_openai(tools: &[ToolSpec]) -> Vec { #[async_trait] impl LlmProvider for OpenAi { fn name(&self) -> &str { - "openai" + self.provider_name + } + + fn supports_image_input(&self) -> bool { + self.image_input } async fn complete( @@ -338,3 +369,62 @@ fn short_hash(s: &str) -> String { s.hash(&mut h); format!("{:x}", h.finish())[..8].to_string() } + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider::ImageInput; + + #[test] + fn user_images_use_chat_completions_image_url_blocks() { + let messages = [Message { + role: Role::User, + content: Content::UserInput { + text: "describe it".into(), + images: vec![ImageInput { + media_type: "image/png".into(), + data: "aGVsbG8=".into(), + source: Some("/tmp/shot.png".into()), + }], + }, + }]; + let wire = messages_to_openai("", &messages); + assert_eq!( + wire[0]["content"][0], + json!({"type":"text","text":"describe it"}) + ); + assert_eq!(wire[0]["content"][1]["type"], "image_url"); + assert_eq!( + wire[0]["content"][1]["image_url"]["url"], + "data:image/png;base64,aGVsbG8=" + ); + assert!(!wire[0].to_string().contains("/tmp/shot.png")); + } + + #[test] + fn deepseek_dialect_is_explicitly_text_only() { + let provider = + OpenAi::text_only(Some("https://example.invalid".into()), "key".into()).unwrap(); + assert_eq!(provider.name(), "deepseek"); + assert!(!provider.supports_image_input()); + let error = crate::provider::ensure_image_input_supported( + &provider, + &[Message { + role: Role::User, + content: Content::UserInput { + text: "image".into(), + images: vec![ImageInput { + media_type: "image/png".into(), + data: "YWJj".into(), + source: None, + }], + }, + }], + None, + ) + .unwrap_err(); + let error = error.to_string(); + assert!(error.contains("deepseek does not support image input")); + assert!(error.contains("use /reset before continuing")); + } +} diff --git a/crates/adapter-smith/src/provider_watchdog.rs b/crates/adapter-smith/src/provider_watchdog.rs index 6b7bde63..8d19aa82 100644 --- a/crates/adapter-smith/src/provider_watchdog.rs +++ b/crates/adapter-smith/src/provider_watchdog.rs @@ -36,6 +36,7 @@ pub async fn complete( tools: &[ToolSpec], sink: &mut dyn TextSink, ) -> Result { + crate::provider::ensure_image_input_supported(provider, messages, None)?; let timeout = provider_idle_timeout(); let retry = RetryConfig { max_attempts: provider_retry_attempts(), diff --git a/docs/smith.md b/docs/smith.md index 59259c3f..8db8fc7c 100644 --- a/docs/smith.md +++ b/docs/smith.md @@ -175,6 +175,28 @@ Profiles are always referenced with the explicit `@` prefix; bare names never resolve to a profile. The status line shows `@:` so you can tell which endpoint is active. +### Image input + +Smith accepts PNG, JPEG, GIF, and WebP images as native multimodal user input. +In the TUI, use `C-x v` (or `/paste`) to paste a clipboard image, drop an image +file into the terminal, or include an explicit local path in the prompt: + +```text +Describe ./screenshots/error.png +Compare [#file:/path/to/session/attachments/before.png] with ./after.jpg +``` + +Construct stores pasted/uploaded bytes in the session first; Smith snapshots +the image into its persisted conversation when the prompt is submitted. The +same image therefore survives daemon restart and provider retry even if the +source file later changes. Limits are 5 MiB per image and 20 images per turn. + +OpenAI, Anthropic (including Claude/Kimi OAuth), Gemini, Meta, Codex OAuth, +OpenRouter/Grok, and Ollama wires receive provider-native image content. The +selected model must itself support vision. DeepSeek's current chat endpoint is +text-only, so Smith rejects image turns locally and asks you to switch +providers; it does not silently send a path as plain text. + ### Tools Smith registers three tool sets: local development tools, Chrome DevTools browser automation, and daemon/fleet control tools (including subagents). diff --git a/specs/0142-smith-meta-model-api.md b/specs/0142-smith-meta-model-api.md index 05bbd9ea..53a5b2b3 100644 --- a/specs/0142-smith-meta-model-api.md +++ b/specs/0142-smith-meta-model-api.md @@ -27,5 +27,5 @@ Muse Spark 1.1 advertises a one-million-token context window, so Smith starts wi ## Non-Goals -- Meta-hosted search, media inputs, and other provider-native tools are not exposed by this initial integration. +- Meta-hosted search and provider-native tools are not exposed by this integration. User image content follows Smith's cross-provider image-input contract. - Reasoning traces are not surfaced or persisted unless Meta exposes a replayable and user-visible reasoning format. diff --git a/specs/0214-smith-image-input-is-provider-native.md b/specs/0214-smith-image-input-is-provider-native.md new file mode 100644 index 00000000..ac1fbaa0 --- /dev/null +++ b/specs/0214-smith-image-input-is-provider-native.md @@ -0,0 +1,62 @@ +# 0214-smith-image-input-is-provider-native + +Status: accepted +Date: 2026-09-09 +Area: harness +Scope: Smith accepts explicit local image references as durable multimodal user turns and translates them at each provider boundary. + +## Decision + +Smith recognizes explicit local image input in submitted user text: a bare or +relative image path, a Construct `[#file:…]` session-attachment reference, or a +local Markdown image link. Clipboard images and remotely uploaded files use the +same path because Construct clients first store their bytes as session +attachments and paste the resulting daemon-host path into the harness. + +At submit time Smith reads each image, validates its format and portable size +limits, and snapshots its MIME type and base64 bytes into the canonical user +message alongside the original text. That canonical message is what Smith +persists and replays. Resume and provider retries therefore use the same bytes; +they do not depend on the source path continuing to exist or retaining the same +contents. + +Provider adapters translate the canonical image into their native wire shape: +Chat Completions image URL blocks, Anthropic image source blocks, Gemini inline +data parts, Responses API input-image blocks, or Ollama image arrays. Text-only +turns retain their existing message variant and wire shape. A wire-capable +provider may still report that a selected model lacks vision. A provider whose +wire is known not to support image input must reject the turn locally with a +clear error before the new turn is persisted; DeepSeek is currently in that +category. + +Portable inputs are PNG, JPEG, GIF, and WebP, at most 5 MiB per image and 20 +images per turn. Local paths are never fetched from remote URLs. + +## Reason + +Clients already had a secure, session-scoped way to carry pasted binary data +to the daemon host, but Smith treated the resulting path as ordinary text. The +model could only discover the image by choosing a filesystem tool, and several +providers never received their native multimodal content at all. Normalizing +once at the Smith boundary keeps clients provider-agnostic while preserving +the exact turn across retries, restarts, and model changes. + +## Consequences + +- The structured transcript remains textual and shows exactly what the user + submitted; image bytes live only in Smith's persisted canonical history and + provider requests. +- Compaction describes the count of images in compacted history rather than + embedding their base64 in the summarizer prompt. Normal context pruning drops + an image with its complete user-led turn. +- Switching an image-bearing conversation to a text-only provider fails + clearly until the user switches back or clears that conversation history. +- Provider implementations added later must explicitly advertise and implement + image translation before Smith will send multimodal history to them. + +## Non-Goals + +- PDF, video, audio, remote-URL, or model-generated image support. +- Inferring that incidental prose mentioning an image filename is an + attachment; references must have an explicit path-like shape. +- A new image preview or attachment chip in Smith's transcript UI.