diff --git a/AGENTS.md b/AGENTS.md index 0d34971a..b0a068c8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ Multi-crate Rust workspace for Workshop, a Host, and the tooling that builds it Seven terms have exactly one meaning each, everywhere in this repository: code comments, docs, rulebooks, and plans. Write them capitalized. -- **Engine**: the `runvil-*` crates. The Engine parses a prompt and steps a run. Whenever a run needs a model reply, a Pack's render of the tools a section offers it, a Pack tool's result, a timer, or a store operation, the Engine emits an effect and waits for its caller to answer it. In production the caller is the Harness. +- **Engine**: the `runvil-*` crates. The Engine parses a prompt and steps a run. Whenever a run needs a model reply, a Pack's render of the tools a section offers it, a Pack tool's result, a timer, or a file operation, the Engine emits an effect and waits for its caller to answer it. In production the caller is the Harness. - **Harness**: the `runvil` crate, which is the one crate a Host depends on, and the `harness-*` crates. The Harness steps the Engine, performs every effect, returns each answer, and records every run through the recorder the Host supplies. Production code runs prompts only through the Harness. - **Host**: an application that runs prompts through the Harness, such as Workshop or Papergate. The Host makes every policy decision. It may also use the Engine's parser and types to read prompts and show events. - **Pack**: code which installs an optional native prelude, an optional Lua prelude, and zero or more tools. diff --git a/Cargo.lock b/Cargo.lock index 8f13c51f..12b665fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2242,7 +2242,7 @@ dependencies = [ [[package]] name = "harness-gateway-client" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "async-trait", "bytes", @@ -2258,7 +2258,7 @@ dependencies = [ [[package]] name = "harness-plugin-loader" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "libloading 0.8.9", "runvil-plugin", @@ -2269,7 +2269,7 @@ dependencies = [ [[package]] name = "harness-runner" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "futures-util", "rand 0.9.5", @@ -4052,7 +4052,7 @@ dependencies = [ [[package]] name = "plugin-bashkit" version = "0.1.0" -source = "git+https://github.com/cppalliance/runvil-plugin-bashkit?rev=0864f1ff9cc1f4fad6c96f1e9f6d8a3ab85e9580#0864f1ff9cc1f4fad6c96f1e9f6d8a3ab85e9580" +source = "git+https://github.com/cppalliance/runvil-plugin-bashkit?rev=f348f7a67b9087bcbc55ea7cb61c8de3fc409c82#f348f7a67b9087bcbc55ea7cb61c8de3fc409c82" dependencies = [ "bashkit", "runvil-plugin-sdk", @@ -4064,7 +4064,7 @@ dependencies = [ [[package]] name = "plugin-mcp" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "reqwest", "rmcp", @@ -4077,7 +4077,7 @@ dependencies = [ [[package]] name = "plugin-std" version = "0.1.0" -source = "git+https://github.com/cppalliance/runvil-plugin-std?rev=782ad23e5dfbe6fbfa5be7145fa73a9ca60ce8a8#782ad23e5dfbe6fbfa5be7145fa73a9ca60ce8a8" +source = "git+https://github.com/cppalliance/runvil-plugin-std?rev=a806bf63e3b0a23c85f3599a1b03fea578c77a2d#a806bf63e3b0a23c85f3599a1b03fea578c77a2d" dependencies = [ "regex", "runvil-plugin-sdk", @@ -4088,7 +4088,7 @@ dependencies = [ [[package]] name = "plugin-web" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "async-trait", "encoding_rs", @@ -4734,7 +4734,7 @@ dependencies = [ [[package]] name = "runvil" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "harness-plugin-loader", "harness-runner", @@ -4745,7 +4745,7 @@ dependencies = [ [[package]] name = "runvil-engine" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "runvil-lua", "runvil-model-client", @@ -4758,7 +4758,7 @@ dependencies = [ [[package]] name = "runvil-lua" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "mlua", "runvil-markdown", @@ -4772,7 +4772,7 @@ dependencies = [ [[package]] name = "runvil-markdown" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "pulldown-cmark", "serde_json", @@ -4782,7 +4782,7 @@ dependencies = [ [[package]] name = "runvil-model-client" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "runvil-types", "serde", @@ -4792,7 +4792,7 @@ dependencies = [ [[package]] name = "runvil-plugin" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "runvil-types", "runvil-vfs", @@ -4801,21 +4801,21 @@ dependencies = [ [[package]] name = "runvil-plugin-sdk" -version = "0.4.0" -source = "git+https://github.com/cppalliance/runvil-plugin-api?rev=03a9fa37d2fe690d42ee0aa5c7a8289173e8e0f2#03a9fa37d2fe690d42ee0aa5c7a8289173e8e0f2" +version = "0.5.0" +source = "git+https://github.com/cppalliance/runvil-plugin-api?rev=52a4905aca0a9c945a93dc4ee7ea815db85612e4#52a4905aca0a9c945a93dc4ee7ea815db85612e4" dependencies = [ "runvil-plugin-sys", ] [[package]] name = "runvil-plugin-sys" -version = "0.4.0" -source = "git+https://github.com/cppalliance/runvil-plugin-api?rev=03a9fa37d2fe690d42ee0aa5c7a8289173e8e0f2#03a9fa37d2fe690d42ee0aa5c7a8289173e8e0f2" +version = "0.5.0" +source = "git+https://github.com/cppalliance/runvil-plugin-api?rev=52a4905aca0a9c945a93dc4ee7ea815db85612e4#52a4905aca0a9c945a93dc4ee7ea815db85612e4" [[package]] name = "runvil-prompt" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "pulldown-cmark", "runvil-lua", @@ -4828,7 +4828,7 @@ dependencies = [ [[package]] name = "runvil-runtime" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "mlua", "regex", @@ -4846,7 +4846,7 @@ dependencies = [ [[package]] name = "runvil-types" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" dependencies = [ "serde", "serde_json", @@ -4856,7 +4856,7 @@ dependencies = [ [[package]] name = "runvil-vfs" version = "0.3.0" -source = "git+https://github.com/cppalliance/runvil-harness?rev=1664eaf363569d629d8ec65f3e0381bded3449bd#1664eaf363569d629d8ec65f3e0381bded3449bd" +source = "git+https://github.com/cppalliance/runvil-harness?rev=4afed45fe5a4198ee9c9617396377d83b88f5e16#4afed45fe5a4198ee9c9617396377d83b88f5e16" [[package]] name = "rust-embed" diff --git a/Cargo.toml b/Cargo.toml index 50b0d05f..6252e917 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,29 +25,29 @@ repository = "https://github.com/cppalliance/runvil-workshop" # in. All four name one commit, because two commits would be two copies # of every type the crates share. `version` beside `git` keeps cargo-deny's # wildcard check quiet. -runvil = { version = "0.3.0", git = "https://github.com/cppalliance/runvil-harness", rev = "1664eaf363569d629d8ec65f3e0381bded3449bd" } -harness-gateway-client = { version = "0.3.0", git = "https://github.com/cppalliance/runvil-harness", rev = "1664eaf363569d629d8ec65f3e0381bded3449bd" } -plugin-web = { version = "0.3.0", git = "https://github.com/cppalliance/runvil-harness", rev = "1664eaf363569d629d8ec65f3e0381bded3449bd" } -plugin-mcp = { version = "0.3.0", git = "https://github.com/cppalliance/runvil-harness", rev = "1664eaf363569d629d8ec65f3e0381bded3449bd" } -# The safe layer over the C Plugin contract, version 0.4, from its own +runvil = { version = "0.3.0", git = "https://github.com/cppalliance/runvil-harness", rev = "4afed45fe5a4198ee9c9617396377d83b88f5e16" } +harness-gateway-client = { version = "0.3.0", git = "https://github.com/cppalliance/runvil-harness", rev = "4afed45fe5a4198ee9c9617396377d83b88f5e16" } +plugin-web = { version = "0.3.0", git = "https://github.com/cppalliance/runvil-harness", rev = "4afed45fe5a4198ee9c9617396377d83b88f5e16" } +plugin-mcp = { version = "0.3.0", git = "https://github.com/cppalliance/runvil-harness", rev = "4afed45fe5a4198ee9c9617396377d83b88f5e16" } +# The safe layer over the C Plugin contract, version 0.5, from its own # repository, pinned by commit. Only the library Plugin crates under # `crates/workshop/` name it, for `export_plugin!`. The commit is the one # runvil-harness pins the contract's `runvil-plugin-sys` at: that crate # sets `links`, so Cargo refuses two commits of the contract in one build. -runvil-plugin-sdk = { version = "0.4.0", git = "https://github.com/cppalliance/runvil-plugin-api", rev = "03a9fa37d2fe690d42ee0aa5c7a8289173e8e0f2" } +runvil-plugin-sdk = { version = "0.5.0", git = "https://github.com/cppalliance/runvil-plugin-api", rev = "52a4905aca0a9c945a93dc4ee7ea815db85612e4" } # The std Plugin, from its own repository, pinned by commit, which must be # built against the contract commit above. default-features off drops its # own export of `runvil_plugin`, which `workshop-plugin-std` makes instead. # The key is `std-plugin` because that crate's own library is named # `plugin_std`, the name of the file Workshop ships. -std-plugin = { package = "plugin-std", version = "0.1.0", git = "https://github.com/cppalliance/runvil-plugin-std", rev = "782ad23e5dfbe6fbfa5be7145fa73a9ca60ce8a8", default-features = false } +std-plugin = { package = "plugin-std", version = "0.1.0", git = "https://github.com/cppalliance/runvil-plugin-std", rev = "a806bf63e3b0a23c85f3599a1b03fea578c77a2d", default-features = false } # The bashkit Plugin, from its own repository, pinned by commit and built # against the same contract commit. default-features off drops its own # export of `runvil_plugin`, which `workshop-plugin-bashkit` makes # instead. The key is `bashkit-plugin` for the reason std's is # `std-plugin`: the crate's own library has the name of the file Workshop # ships, `plugin_bashkit`. -bashkit-plugin = { package = "plugin-bashkit", version = "0.1.0", git = "https://github.com/cppalliance/runvil-plugin-bashkit", rev = "0864f1ff9cc1f4fad6c96f1e9f6d8a3ab85e9580", default-features = false } +bashkit-plugin = { package = "plugin-bashkit", version = "0.1.0", git = "https://github.com/cppalliance/runvil-plugin-bashkit", rev = "f348f7a67b9087bcbc55ea7cb61c8de3fc409c82", default-features = false } workshop-gateway-discovery = { path = "crates/workshop/gateway-discovery" } workshop-server = { path = "crates/workshop/server" } workshop-server-api = { path = "crates/workshop/server-api" } diff --git a/crates/workshop/agents/src/conversation-run.rs b/crates/workshop/agents/src/conversation-run.rs index 77340103..e5cc07d8 100644 --- a/crates/workshop/agents/src/conversation-run.rs +++ b/crates/workshop/agents/src/conversation-run.rs @@ -114,7 +114,7 @@ impl Conversation { /// The agent `name` parsed on `host` on the blocking pool, since finding /// it reads the agents folder, or the message of why it did not parse: the /// parse failure's located message, or the blocking task's failure. -async fn parse(host: &Arc, name: &str) -> Result { +async fn parse(host: &Arc, name: &str) -> Result, String> { let host = Arc::clone(host); let name = name.to_owned(); match tokio::task::spawn_blocking(move || host.parse(&name)).await { diff --git a/crates/workshop/server/src/agents.rs b/crates/workshop/server/src/agents.rs index d50bc9e9..dd06be9a 100644 --- a/crates/workshop/server/src/agents.rs +++ b/crates/workshop/server/src/agents.rs @@ -24,7 +24,11 @@ //! sends each round to the dropdown's current pick. Each run's filesystem //! is an in-memory root, with a read-only mount beside it for each of two //! demos whose switch is on: the language guide, and the machine's C -//! drive (`language_guide`). Both switches are committed as off. The run's +//! drive (`language_guide`). Both switches are committed as off. The +//! filesystem has no policy of its own, so the default one, `AllowAll`, +//! admits every operation: the file tools and the shell change the files +//! of the in-memory root, which end with the run, and a read-only mount +//! refuses each change. The run's //! setup carries the Host state ([`bindings`]): the menu's pick and the //! workspace's granted roots, read once the menu's catalog holds a //! chat-capable model. Status-bar reporting stays in the server diff --git a/crates/workshop/server/src/agents/language_guide/tests.rs b/crates/workshop/server/src/agents/language_guide/tests.rs index e2731f21..34334e1b 100644 --- a/crates/workshop/server/src/agents/language_guide/tests.rs +++ b/crates/workshop/server/src/agents/language_guide/tests.rs @@ -1,12 +1,14 @@ //! Mapping a local directory into a run's filesystem: a directory mapped //! under `/_runvil` reads through the mapping, is reached by listing down //! from the root, refuses writes, leaves the in-memory root writable, and -//! is visible to a prompt that runs over the filesystem. Two directories -//! mapped one after the other are both visible, as the guide and the C -//! drive are with both switches on. Every test uses a local directory it -//! makes, so none depends on the language guide's files or on a path on -//! any one machine, but for the one that maps a drive's root, which runs -//! on Windows alone. +//! is visible to a prompt that runs over the filesystem. The file tools +//! and the shell of such a prompt change nothing under the mapping, and +//! the shell still writes a file of the in-memory root beside it. Two +//! directories mapped one after the other are both visible, as the guide +//! and the C drive are with both switches on. Every test uses a local +//! directory it makes, so none depends on the language guide's files or +//! on a path on any one machine, but for the one that maps a drive's +//! root, which runs on Windows alone. use std::error::Error; use std::path::Path; @@ -243,6 +245,123 @@ async fn a_prompt_sees_the_directory_mapped_under_the_runvil_mount() -> Outcome Ok(()) } +/// A prompt that asks the three `std` tools that change files to change +/// the mapped directory, and returns each refusal on a line of its own. +const CHANGING_PROMPT: &str = r"--- +name: changes-the-mapping +description: tries to change a mapped directory +runvil: 0 +uses: + - std +--- + +## Only + +```lua +local function refused(tool, arguments) + local ok, answer = pcall(tools.call, tool, arguments) + return ok and 'changed' or answer.message +end +return table.concat({ + refused('std/write', { path = '/_runvil/test/new.md', contents = 'new' }), + refused('std/replace', { + path = '/_runvil/test/hello.md', old_string = 'Hello', new_string = 'Goodbye', + }), + refused('std/rm', { path = '/_runvil/test/hello.md' }), +}, '\n') +``` +"; + +#[tokio::test] +async fn the_file_tools_change_nothing_under_the_mapping() -> Outcome { + let directory = local_directory()?; + let (plugins, _library) = plugin_dir("changes-the-mapping"); + let (host, _) = host_packs(&Registry::new(), None, Some(plugins.path())); + + let result = run_prompt_over( + host, + CHANGING_PROMPT.to_owned(), + filesystem_with(directory.path()), + ) + .await; + + let Ok(Ok(final_text)) = result else { + return Err(format!("the prompt did not complete: {result:?}").into()); + }; + let refusals: Vec<&str> = final_text.lines().collect(); + assert_eq!(refusals.len(), 3, "one line for each tool: {final_text}"); + for (refusal, path) in refusals.iter().zip([ + "/_runvil/test/new.md", + "/_runvil/test/hello.md", + "/_runvil/test/hello.md", + ]) { + assert!( + refusal.ends_with(&format!("{path} is served read-only and cannot change")), + "the tool answers with the mapping's refusal: {refusal}" + ); + } + let page = directory.path().join(PAGE); + let text = + std::fs::read_to_string(&page).map_err(|e| format!("read {}: {e}", page.display()))?; + assert_eq!(text, TEXT, "the page is as it was"); + assert!(!directory.path().join("new.md").exists()); + Ok(()) +} + +/// A prompt that asks the shell to make a file in the mapped directory, +/// to remove its page, and to make a file in the in-memory root, and +/// returns what the shell answers. +const CHANGING_SHELL_PROMPT: &str = r#"--- +name: shells-into-the-mapping +description: tries to change a mapped directory with the shell +runvil: 0 +uses: [bash] +--- + +## Only + +```lua +return tools.call('bash/shell', { + command = 'echo new > /_runvil/test/new.md; echo "write $?"; ' + .. 'rm /_runvil/test/hello.md; echo "remove $?"; ' + .. 'echo kept > /scratch.txt; cat /scratch.txt', +}) +``` +"#; + +#[tokio::test] +async fn the_shell_changes_nothing_under_the_mapping_and_writes_beside_it() -> Outcome { + let directory = local_directory()?; + let vfs = filesystem_with(directory.path()); + let (plugins, _library) = plugin_dir_of("shells-into-the-mapping", BASHKIT_LIBRARY); + let (host, _) = host_packs(&Registry::new(), None, Some(plugins.path())); + + let result = run_prompt_over(host, CHANGING_SHELL_PROMPT.to_owned(), vfs.clone()).await; + + let Ok(Ok(final_text)) = result else { + return Err(format!("the prompt did not complete: {result:?}").into()); + }; + assert!( + final_text.contains("```\nwrite 1\nremove 1\nkept\n"), + "the write and the removal under the mapping both fail, and the shell writes and \ + reads a file of the in-memory root: {final_text}" + ); + for path in ["/_runvil/test/new.md", "/_runvil/test/hello.md"] { + assert!( + final_text.contains(&format!("{path} is served read-only and cannot change\n")), + "the shell prints the mapping's refusal of {path}: {final_text}" + ); + } + let page = directory.path().join(PAGE); + let text = + std::fs::read_to_string(&page).map_err(|e| format!("read {}: {e}", page.display()))?; + assert_eq!(text, TEXT, "the page is as it was"); + assert!(!directory.path().join("new.md").exists()); + let access = vfs.acquire(Origin::new("check"))?; + assert_eq!(access.read_string("/scratch.txt")?, "kept\n"); + Ok(()) +} + /// A prompt that lists the root and reads a page of each mapped /// directory with the shell, and returns what the shell answers. const SHELL_PROMPT: &str = r"--- diff --git a/crates/workshop/server/src/agents/tests.rs b/crates/workshop/server/src/agents/tests.rs index e7024141..176b9a26 100644 --- a/crates/workshop/server/src/agents/tests.rs +++ b/crates/workshop/server/src/agents/tests.rs @@ -12,12 +12,16 @@ //! servers value is the mcp Pack's configuration, so one it refuses //! refuses a prompt requiring `mcp` as it parses. The Host's finder //! refuses an unknown agent as `NotFound` and an unreadable one as `Io`, -//! each with the launch refusal's own text. A configured Plugin directory +//! each with the launch refusal's own text, and it reads an agent that +//! another names under `prompts:`, which that agent then runs with +//! `execute`. A configured Plugin directory //! loads each library Plugin in it and reports each file that is none, -//! and a directory that does not exist loads nothing. A prompt reads a -//! file of its run through the std library's `Read` tool and through the -//! bashkit library's `Shell` tool, and a Host with no Plugin directory -//! refuses a prompt that requires `std` or `bash`. +//! and a directory that does not exist loads nothing. A prompt reads and +//! changes the files of its run through the std library's `Read`, +//! `Write`, `StrReplace`, and `Delete` tools and through the bashkit +//! library's `Shell` tool, and the Host reads each change back after the +//! run. A Host with no Plugin directory refuses a prompt that requires +//! `std` or `bash`. use std::env::consts::{DLL_EXTENSION, DLL_PREFIX, DLL_SUFFIX}; use std::path::PathBuf; @@ -85,24 +89,35 @@ async fn run_prompt(packs: Packs, source: String) -> Result Result, String> { - let recorder = Arc::new(MemoryRecorder::new()); - let conversation = Conversations::new().open("browses"); - let mut errors = conversation.subscribe_errors(); - let host = Arc::new(Host { + let host = Host { packs, prompts: Arc::new(move |_: &str| -> Result { Ok(source.clone()) }), timer: Arc::new(TokioTimer), - }); + }; + run_agent_on(host, "browses", vfs).await +} + +/// Runs the agent `name` of `host` as one conversation, which has a user, +/// over the filesystem `vfs`, then reads the conversation's state and how +/// it ended: the result of its one run, or the message of the run-failed +/// report when the Host refused the prompt as it parsed it and no run +/// began. +async fn run_agent_on( + host: Host, + name: &str, + vfs: VfsRef, +) -> Result, String> { + let recorder = Arc::new(MemoryRecorder::new()); + let conversation = Conversations::new().open(name); + let mut errors = conversation.subscribe_errors(); let setup = RunSetup { vfs, has_user: true, @@ -113,7 +128,7 @@ pub(super) async fn run_prompt_over( }; tokio::time::timeout( Duration::from_secs(10), - conversation.run(host, "browses", setup), + conversation.run(Arc::new(host), name, setup), ) .await .expect("the run ends in time"); @@ -225,6 +240,32 @@ fn the_finder_refuses_an_unknown_agent_as_not_found_and_an_unreadable_one_as_io( } } +#[tokio::test] +async fn an_agent_executes_an_agent_of_the_folder_that_its_prompts_list_names() { + let dir = tempfile::TempDir::new().expect("tempdir"); + // A `prompts:` entry is the name the finder knows the agent under, + // which is the name discovery lists: the file's stem. + let outer = "---\nname: outer\ndescription: runs inner\nrunvil: 0\n\ + prompts:\n - inner\n---\n\n## Only\n\n```lua\n\ + return 'outer got ' .. execute('inner', {})\n```\n"; + let inner = "---\nname: inner\ndescription: answers\nrunvil: 0\n---\n\n\ + ## Only\n\n```lua\nreturn 'inner ran'\n```\n"; + std::fs::write(dir.path().join("outer.md"), outer).expect("write fixture"); + std::fs::write(dir.path().join("inner.md"), inner).expect("write fixture"); + let (packs, _) = host_packs(&Registry::new(), None, None); + let host = Host { + packs, + prompts: Arc::new(agent_finder(dir.path().to_path_buf())), + timer: Arc::new(TokioTimer), + }; + + assert_eq!( + run_agent_on(host, "outer", VfsRef::default()).await, + Ok(Ok("outer got inner ran".to_owned())), + "the Host's finder reads the listed agent, and `execute` runs it in the same run" + ); +} + #[tokio::test] async fn a_prompt_declaring_mcp_prepares_on_the_runtime_the_server_provides() { // The server's own `services` provides the runtime, and `plugin-mcp` @@ -414,33 +455,44 @@ fn only_a_plugin_directory_with_a_shipped_library_gives_the_host_its_pack() { } } -/// A prompt that declares `std` and returns what `std/read` answers for -/// `/notes.md`, then what `std/write` answers. -const READS_NOTES: &str = "---\nname: reads\ndescription: reads a file\nrunvil: 0\n\ - uses:\n - std\n---\n\n## Only\n\n```lua\n\ +/// A prompt that declares `std` and returns what four of its tools +/// answer, in this order: `std/read` for `/notes.md`, `std/write` for a +/// file under a directory that is absent, `std/replace` for a text of +/// `/notes.md`, and `std/rm` for `/old.md`. +const READS_AND_CHANGES: &str = "---\nname: changes\ndescription: reads and changes files\n\ + runvil: 0\nuses:\n - std\n---\n\n## Only\n\n```lua\n\ local read = tools.call('std/read', { path = '/notes.md' })\n\ - local write = tools.call('std/write', { path = '/out.md', contents = 'x' })\n\ - return read .. '|' .. write\n```\n"; + local write = tools.call('std/write', { path = '/out/new.md', contents = 'x' })\n\ + local replace = tools.call('std/replace', {\n\ + path = '/notes.md', old_string = 'world', new_string = 'there',\n\ + })\n\ + local delete = tools.call('std/rm', { path = '/old.md' })\n\ + return read .. '|' .. write .. '|' .. replace .. '|' .. delete\n```\n"; #[tokio::test] -async fn a_prompt_reads_a_file_of_its_run_through_the_std_library_plugin() { - let (dir, _library) = plugin_dir("reads-notes"); +async fn a_prompt_reads_and_changes_files_of_its_run_through_the_std_library_plugin() { + let (dir, _library) = plugin_dir("reads-and-changes"); let (host, report) = host_packs(&Registry::new(), None, Some(dir.path())); let report = report.expect("the configured directory is read"); assert!(report.failures().is_empty(), "{:?}", report.failures()); let vfs = VfsRef::default(); - vfs.acquire(Origin::new("seed")) - .expect("a fresh memory filesystem acquires") - .write("/notes.md", b"hello\nworld\n") + let seed = vfs + .acquire(Origin::new("seed")) + .expect("a fresh memory filesystem acquires"); + seed.write("/notes.md", b"hello\nworld\n") .expect("the file is seeded"); + seed.write("/old.md", b"old\n").expect("the file is seeded"); - let result = run_prompt_over(host, READS_NOTES.to_owned(), vfs).await; + let result = run_prompt_over(host, READS_AND_CHANGES.to_owned(), vfs.clone()).await; let Ok(Ok(text)) = result else { panic!("the run ends with its text: {result:?}"); }; - // Each numbered line holds a `|`, and the stub's text holds none. - let (read, write) = text.rsplit_once('|').expect("the prompt joins two answers"); + // Each numbered line holds a `|`, and the three trusted texts hold + // none, so the answers split from the right. + let mut answers = text.rsplitn(4, '|'); + let mut next = || answers.next().expect("the prompt joins four answers"); + let (delete, replace, write, read) = (next(), next(), next(), next()); assert!( read.contains(" 1|hello\n 2|world\n"), "Read answers the file's numbered lines: {read}" @@ -450,11 +502,27 @@ async fn a_prompt_reads_a_file_of_its_run_through_the_std_library_plugin() { "file text reaches the script marked as untrusted: {read}" ); assert_eq!( - write, - "The Write tool is not implemented in this build. The implementation is missing and \ - nothing in this session will change that. Calling Write again is safe, but it will \ - return this same result every time, whatever the arguments. Carry on without it.", - "Write is still a stub, and its text is trusted, so it comes back as it is" + (write, replace, delete), + ("Wrote 1 byte.", "Made 1 replacement.", "Deleted the file."), + "each tool that changes files says what it did, as trusted text" + ); + + let check = vfs + .acquire(Origin::new("check")) + .expect("the filesystem acquires"); + assert_eq!( + check.read_string("/out/new.md").expect("the file is read"), + "x", + "Write made the file and the directory on the way to it" + ); + assert_eq!( + check.read_string("/notes.md").expect("the file is read"), + "hello\nthere\n", + "StrReplace changed the one text" + ); + assert!( + !check.exists("/old.md").expect("the filesystem answers"), + "Delete removed the file" ); } @@ -478,10 +546,13 @@ async fn without_a_plugin_directory_a_prompt_requiring_a_shipped_pack_is_refused } /// A prompt that declares `bash` and returns what `bash/shell` answers -/// for a command that reads `/notes.md` and then writes a file. +/// for a command that reads `/notes.md`, writes a file under a directory +/// it makes, and edits `/notes.md` in place. const RUNS_A_COMMAND: &str = "---\nname: shells\ndescription: runs a command\nrunvil: 0\n\ uses:\n - bash\n---\n\n## Only\n\n```lua\n\ - return tools.call('bash/shell', { command = 'cat /notes.md; echo x > /out.md' })\n```\n"; + return tools.call('bash/shell', {\n\ + command = 'cat /notes.md; mkdir /out; echo x > /out/new.md; sed -i s/world/there/ /notes.md',\n\ + })\n```\n"; #[tokio::test] async fn a_prompt_runs_a_command_over_its_runs_files_through_the_bashkit_library_plugin() { @@ -511,20 +582,23 @@ async fn a_prompt_runs_a_command_over_its_runs_files_through_the_bashkit_library "a command's output reaches the script marked as untrusted: {text}" ); assert!( - text.contains("Exit code: 1\n") && text.contains("```\nhello\nworld\n"), - "the shell reads the file through the run's filesystem, and the write fails the \ - command: {text}" - ); - assert!( - text.contains("/out.md: the shell cannot change files in this build"), - "the shell says why the write failed: {text}" + text.contains("Exit code: 0\n\nCommand output:\n\n```\nhello\nworld\n```"), + "the shell reads the file through the run's filesystem, and each change succeeds, \ + so the command prints the file alone: {text}" ); - let written = vfs + let check = vfs .acquire(Origin::new("check")) - .expect("the filesystem acquires") - .exists("/out.md") - .expect("the filesystem answers"); - assert!(!written, "the shell changes no file of the run"); + .expect("the filesystem acquires"); + assert_eq!( + check.read_string("/out/new.md").expect("the file is read"), + "x\n", + "the shell made the directory and the file in the run's filesystem" + ); + assert_eq!( + check.read_string("/notes.md").expect("the file is read"), + "hello\nthere\n", + "the shell edited the seeded file in place" + ); } #[test] diff --git a/crates/workshop/server/src/routes/prompts-tests.rs b/crates/workshop/server/src/routes/prompts-tests.rs index 87e0e5ae..f5bea10a 100644 --- a/crates/workshop/server/src/routes/prompts-tests.rs +++ b/crates/workshop/server/src/routes/prompts-tests.rs @@ -163,6 +163,22 @@ async fn a_prompt_without_args_yields_the_default_declaration() { ); } +#[tokio::test] +async fn a_prompt_that_lists_prompts_answers_its_own_contract() { + let text = "---\nname: lists\ndescription: runs another prompt\n\ + prompts:\n - review.md\n - summarize.md\n---\n\n## S\n\n\ + ```lua\nreturn execute('review.md', {})\n```\n"; + let body = serde_json::json!({ "name": "lists.md", "text": text }); + let (status, json) = post_contract(body).await; + assert_eq!( + status, + StatusCode::OK, + "the listed prompts are not at hand, and the parse still succeeds: {json}" + ); + assert_eq!(json["name"], "lists"); + assert_eq!(json["description"], "runs another prompt"); +} + #[tokio::test] async fn a_lua_error_answers_parse_lua() { let text = "---\nname: badlua\ndescription: bad lua\n---\n\n## S\n\n```lua\nreturn (\n```\n"; diff --git a/crates/workshop/server/src/routes/prompts.rs b/crates/workshop/server/src/routes/prompts.rs index 4b98d698..895a228f 100644 --- a/crates/workshop/server/src/routes/prompts.rs +++ b/crates/workshop/server/src/routes/prompts.rs @@ -121,6 +121,11 @@ impl From<&Frontmatter> for ContractResponse { /// Parses the posted prompt text and answers the contract DTO, or a /// `422` envelope when the text is not a valid prompt. +/// +/// The route has the posted text alone, so its finder gives that text +/// under every name. The parser reads each prompt that a `prompts:` list +/// names from it, so a prompt that lists others parses, and a listed +/// prompt that is missing or malformed is not reported here. async fn contract(Json(body): Json) -> Response { match Prompt::parse(&|_: &str| Ok(body.text.clone()), &body.name) { Ok(prompt) => ( diff --git a/crates/workshop/server/tests/it/chat_gate.rs b/crates/workshop/server/tests/it/chat_gate.rs index 116d1d2e..e7b97bfe 100644 --- a/crates/workshop/server/tests/it/chat_gate.rs +++ b/crates/workshop/server/tests/it/chat_gate.rs @@ -27,6 +27,7 @@ use axum::routing::{get, post}; use futures_util::StreamExt as _; use serde_json::json; +use workshop_run_log::{RunId, RunLog}; use workshop_server::fixtures::{gateway_updater, replace_gateway}; use workshop_server::{AgentsConfig, AppState, Config}; @@ -125,6 +126,18 @@ async fn spawn_chat_server_with_selection(models: &[&str], selected: Option<&str .route("/v1/models", typed_catalog(GATE_MODELS)), ) .await; + spawn_over_gateway(&gateway_url, captured, models, selected).await +} + +/// Spawns the gate server over the gateway at `gateway_url`, whose mock +/// fills `captured`, with `models` in the retained catalog and `selected` +/// picked in the menu. +async fn spawn_over_gateway( + gateway_url: &str, + captured: CapturedRequests, + models: &[&str], + selected: Option<&str>, +) -> GateServer { let dir = tempfile::TempDir::new().expect("tempdir"); let config = Config { agents: AgentsConfig { @@ -132,7 +145,7 @@ async fn spawn_chat_server_with_selection(models: &[&str], selected: Option<&str plugins: Some(plugin_dir()), ..AgentsConfig::default() }, - ..test_config(&gateway_url, dir.path()) + ..test_config(gateway_url, dir.path()) }; let (state, ws_base) = spawn_router(&config).await; state.catalog().publish( @@ -155,6 +168,25 @@ async fn spawn_chat_server_with_selection(models: &[&str], selected: Option<&str } } +/// The run log of `server`, opened once `run` is recorded whole: its row +/// holds how it ended. +async fn closed_run_log(server: &GateServer, run: RunId) -> RunLog { + let log_file = server.dir.path().join("harness").join("runs.db"); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Ok(log) = RunLog::open(&log_file).await + && let Ok(row) = log.run(run).await + && row.result.is_some() + { + return log; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the run is recorded and closed within the deadline") +} + /// Connects to `/agents/ws`, asserting the connect-time list is exactly /// the built-in: end-to-end proof that a missing agents directory still /// offers `chat`. @@ -221,6 +253,7 @@ fn pair(role: &str, content: &str) -> (String, String) { mod canonical_sequence; mod compaction; +mod files; mod lifecycle; mod overload; mod protocol; diff --git a/crates/workshop/server/tests/it/chat_gate/canonical_sequence.rs b/crates/workshop/server/tests/it/chat_gate/canonical_sequence.rs index 0c3ed767..53b103da 100644 --- a/crates/workshop/server/tests/it/chat_gate/canonical_sequence.rs +++ b/crates/workshop/server/tests/it/chat_gate/canonical_sequence.rs @@ -129,8 +129,9 @@ async fn gate_history_accumulates_across_three_turns_byte_exact() { socket.close().await; } -/// GATE 13 - file tools. The chat offers the extra Packs, so its rounds -/// advertise the `std` file tools the model reads the language guide with. +/// GATE 13 - file tools. The chat offers the `std` Pack, so its rounds +/// advertise the file tools: the three the model reads files with, and +/// the three it changes files with. #[tokio::test] async fn gate_a_conversation_round_advertises_the_file_tools() { let server = spawn_chat_server(&["test-model"]).await; @@ -149,7 +150,7 @@ async fn gate_a_conversation_round_advertises_the_file_tools() { .iter() .filter_map(|tool| tool["function"]["name"].as_str()) .collect(); - for wire in ["Read", "ListDir", "Grep"] { + for wire in ["Read", "ListDir", "Grep", "Write", "StrReplace", "Delete"] { assert!( names.contains(&wire), "the conversation round advertises {wire}, got {names:?}" diff --git a/crates/workshop/server/tests/it/chat_gate/compaction.rs b/crates/workshop/server/tests/it/chat_gate/compaction.rs index a223bd31..70fa59ee 100644 --- a/crates/workshop/server/tests/it/chat_gate/compaction.rs +++ b/crates/workshop/server/tests/it/chat_gate/compaction.rs @@ -92,29 +92,7 @@ async fn spawn_compacting_server() -> GateServer { .route("/v1/models", typed_catalog(&["test-model"])), ) .await; - let dir = tempfile::TempDir::new().expect("tempdir"); - let config = Config { - agents: AgentsConfig { - path: dir.path().join("missing-agents"), - plugins: Some(plugin_dir()), - ..AgentsConfig::default() - }, - ..test_config(&gateway_url, dir.path()) - }; - let (state, ws_base) = spawn_router(&config).await; - state - .catalog() - .publish(vec![json!({ "id": "test-model", "object": "model" })]); - state - .menu() - .set_selected("test-model") - .expect("the selected model is in the retained catalog"); - GateServer { - ws_base, - state, - captured, - dir, - } + spawn_over_gateway(&gateway_url, captured, &["test-model"], Some("test-model")).await } /// Collects frames until the `agent_message` event holding `reply` diff --git a/crates/workshop/server/tests/it/chat_gate/files.rs b/crates/workshop/server/tests/it/chat_gate/files.rs new file mode 100644 index 00000000..0902a4de --- /dev/null +++ b/crates/workshop/server/tests/it/chat_gate/files.rs @@ -0,0 +1,264 @@ +//! The file gates: the tools a model calls change the files of the chat's +//! run. The file tools make, edit, and delete a file, and the shell makes +//! one that the file tools read. The transcript and the model's next +//! request hold each answer, and the run log holds each file operation as +//! an effect that names its tool call. The server gives a run's +//! filesystem no policy of its own, so the default one, `AllowAll`, +//! admits each change, and no call here waits for an approval. + +use std::collections::HashSet; + +use workshop_run_log::{RecordFilter, RecordKind}; + +use super::*; +use crate::agents::Turn; + +/// One call of the scripted model: a tool's wire name and its arguments. +type Call = (&'static str, serde_json::Value); + +/// Streams one tool-call round attributed to `model`: the call `id` of +/// the tool with the wire name `name`, with `arguments`. +fn call_round(model: &str, id: &str, name: &str, arguments: &serde_json::Value) -> Response { + let null = serde_json::Value::Null; + let call = json!({ "tool_calls": [{ + "index": 0, "id": id, "type": "function", + "function": { "name": name, "arguments": arguments.to_string() }, + }] }); + let mut sse = String::new(); + for event in [ + sse_chunk(model, &json!({ "role": "assistant" }), &null), + sse_chunk(model, &call, &null), + sse_chunk(model, &json!({}), &json!("tool_calls")), + ] { + sse.push_str(&sse_line(&event)); + } + sse.push_str("data: [DONE]\n\n"); + ([(header::CONTENT_TYPE, "text/event-stream")], sse).into_response() +} + +/// The scripted mock: the first request gets the first of `calls` as a +/// round of one tool call, the second request the second, and every +/// request after the last call streams `echo:done`. Every request body is +/// captured. +fn scripted_completions(captured: &CapturedRequests, calls: &[Call], body: &str) -> Response { + let request: serde_json::Value = serde_json::from_str(body).expect("the request is JSON"); + let model = request["model"].as_str().unwrap_or("test-model").to_owned(); + let round = { + let mut captured = captured.lock().expect("the capture lock is healthy"); + captured.push(request); + captured.len() - 1 + }; + match calls.get(round) { + Some((name, arguments)) => call_round(&model, &format!("call_{round}"), name, arguments), + None => echo_stream(&model, "done"), + } +} + +/// Spawns the gate server over the scripted mock, with `test-model` in +/// the retained catalog and selected in the menu. +async fn spawn_scripted_server(calls: Vec) -> GateServer { + let captured = CapturedRequests::default(); + let mock = Arc::clone(&captured); + let calls = Arc::new(calls); + let gateway_url = spawn_gateway( + Router::new() + .route( + "/v1/chat/completions", + post(move |body: String| { + let (captured, calls) = (Arc::clone(&mock), Arc::clone(&calls)); + async move { scripted_completions(&captured, &calls, &body) } + }), + ) + .route("/v1/models", typed_catalog(&["test-model"])), + ) + .await; + spawn_over_gateway(&gateway_url, captured, &["test-model"], Some("test-model")).await +} + +/// What each tool call of `turn` answered, in order: the content of each +/// `tool_call_update` event, which is what the transcript shows. +fn tool_results(turn: &Turn) -> Vec { + turn.events + .iter() + .filter(|frame| frame["event"]["kind"] == "tool_call_update") + .map(|frame| { + frame["event"]["content"] + .as_str() + .expect("a tool result holds text") + .to_owned() + }) + .collect() +} + +/// The tool message each request after the first ends with, in order: +/// what the model reads as the answer to the call of its last reply. +fn shown_answers(requests: &[serde_json::Value]) -> Vec { + requests + .iter() + .skip(1) + .map(|request| { + let last = request["messages"] + .as_array() + .and_then(|messages| messages.last()) + .expect("a request holds messages"); + assert_eq!(last["role"], "tool", "the request ends with a tool message"); + last["content"] + .as_str() + .expect("a tool message holds text") + .to_owned() + }) + .collect() +} + +/// Sends the operator's `text` to a chat launched on `server` and returns +/// the session with the turn the scripted model's calls and reply make. +async fn run_turn(server: &GateServer, socket: &mut JsonSocket, text: &str) -> (String, Turn) { + let session = launch_chat(socket).await; + next_wait(socket).await; + answer(socket, text).await; + let turn = collect_turn(socket).await; + assert_eq!( + delta_text(&turn), + "echo:done", + "the turn ends with the reply" + ); + assert_eq!( + shown_answers(&server.captured.lock().expect("the capture lock is healthy")), + tool_results(&turn), + "the model reads each answer in its next request as the transcript shows it" + ); + (session, turn) +} + +/// GATE 15 - changing files with the file tools. The model's `Write` +/// makes a file under a directory that is absent, its `StrReplace` edits +/// the file, and its `Delete` removes it: the `Read` after the edit +/// answers the edited text, and the `ListDir` after the removal lists the +/// directory without the file. The run log holds the write as a `Vfs` +/// effect that names the `Write` call, with its answer, and every `Vfs` +/// effect of the run names a tool call. +#[tokio::test] +async fn gate_the_models_file_tool_calls_change_a_file_of_the_run() { + const NOTE: &str = "/notes/todo.md"; + const TEXT: &str = "alpha\nbeta\n"; + let server = spawn_scripted_server(vec![ + ("Write", json!({ "path": NOTE, "contents": TEXT })), + ( + "StrReplace", + json!({ "path": NOTE, "old_string": "beta", "new_string": "BETA" }), + ), + ("Read", json!({ "path": NOTE })), + ("Delete", json!({ "path": NOTE })), + ("ListDir", json!({ "target_directory": "/notes" })), + ]) + .await; + let mut socket = connect_chat(&server.ws_base).await; + let (session, turn) = run_turn(&server, &mut socket, "keep a note").await; + + let results = tool_results(&turn); + assert_eq!(results.len(), 5, "one answer for each call: {results:?}"); + assert_eq!(results[0], "Wrote 11 bytes."); + assert_eq!(results[1], "Made 1 replacement."); + assert!( + results[2].contains(" 1|alpha\n 2|BETA\n"), + "Read answers the edited file: {}", + results[2] + ); + assert_eq!(results[3], "Deleted the file."); + assert!( + results[4].contains("/notes/") && !results[4].contains("todo.md"), + "ListDir lists the directory Write made, without the file Delete removed: {}", + results[4] + ); + + let run = server + .state + .agents() + .run_id(&session) + .expect("the chat's run has begun"); + assert!(server.state.agents().close(&session), "the session closes"); + let records = closed_run_log(&server, run) + .await + .records(run, RecordFilter::default()) + .await + .expect("the records read back"); + let effects: Vec<(u64, &serde_json::Value)> = records + .iter() + .filter(|stored| stored.record.kind == RecordKind::Effect) + .filter_map(|stored| Some((stored.record.effect_id?, &stored.record.payload))) + .collect(); + let calls: HashSet = effects + .iter() + .filter(|(_, payload)| payload.get("ToolCall").is_some()) + .map(|(id, _)| *id) + .collect(); + let write_call = effects + .iter() + .find(|(_, payload)| payload.pointer("/ToolCall/tool") == Some(&json!("std/write"))) + .map(|(id, _)| *id) + .expect("the log holds the Write call"); + let file_effects: Vec<(u64, &serde_json::Value)> = effects + .iter() + .filter_map(|(id, payload)| Some((*id, payload.get("Vfs")?))) + .collect(); + for (_, asked) in &file_effects { + assert!( + asked["call"] + .as_u64() + .is_some_and(|call| calls.contains(&call)), + "each file operation names the tool call that asked for it: {asked}" + ); + } + let written: Vec<&(u64, &serde_json::Value)> = file_effects + .iter() + .filter(|(_, asked)| asked["call"] == json!(write_call)) + .collect(); + let [(write_effect, asked)] = written.as_slice() else { + panic!("the Write call asks for one file operation: {written:?}"); + }; + assert_eq!( + **asked, + json!({ "call": write_call, "op": { "Write": { "path": NOTE, "contents": TEXT } } }), + "the effect holds the operation with the text the model wrote" + ); + let answered = records.iter().any(|stored| { + stored.record.kind == RecordKind::Answer + && stored.record.effect_id == Some(*write_effect) + && stored.record.payload.pointer("/VfsReceipt/Ok").is_some() + }); + assert!(answered, "the log holds the answer of the write, a success"); + socket.close().await; +} + +/// GATE 16 - changing files with the shell. The model's `Shell` call +/// makes a file under a directory that is absent and reads it back, and +/// the file tools read the same file, because both Packs work in the +/// files of the one run. +#[tokio::test] +async fn gate_the_models_shell_call_makes_a_file_the_file_tools_read() { + const FILE: &str = "/work/a.txt"; + let server = spawn_scripted_server(vec![ + ( + "Shell", + json!({ "command": format!("echo hello > {FILE}; cat {FILE}") }), + ), + ("Read", json!({ "path": FILE })), + ]) + .await; + let mut socket = connect_chat(&server.ws_base).await; + let (_session, turn) = run_turn(&server, &mut socket, "make a file").await; + + let results = tool_results(&turn); + assert_eq!(results.len(), 2, "one answer for each call: {results:?}"); + assert!( + results[0].contains("Exit code: 0\n\nCommand output:\n\n```\nhello\n```"), + "the shell writes the file and reads it back: {}", + results[0] + ); + assert!( + results[1].contains(" 1|hello\n"), + "Read answers the file the shell made: {}", + results[1] + ); + socket.close().await; +} diff --git a/crates/workshop/server/tests/it/chat_gate/lifecycle.rs b/crates/workshop/server/tests/it/chat_gate/lifecycle.rs index 10f992ef..cfaf0c15 100644 --- a/crates/workshop/server/tests/it/chat_gate/lifecycle.rs +++ b/crates/workshop/server/tests/it/chat_gate/lifecycle.rs @@ -1,27 +1,14 @@ //! Run lifecycle gates: the dropdown's pick serving each next round, and //! delayed startup convergence on the server broker's catalog wait. -use workshop_run_log::{RecordFilter, RecordKind, RunId, RunLog}; +use workshop_run_log::{RecordFilter, RecordKind}; use super::*; /// The model each answered round of `run` names in the run log, in /// round order, read once the run is recorded whole. async fn answered_models(server: &GateServer, run: RunId) -> Vec { - let log_file = server.dir.path().join("harness").join("runs.db"); - let log = tokio::time::timeout(Duration::from_secs(10), async { - loop { - if let Ok(log) = RunLog::open(&log_file).await - && let Ok(row) = log.run(run).await - && row.result.is_some() - { - return log; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("the run is recorded and closed within the deadline"); + let log = closed_run_log(server, run).await; let answers = RecordFilter { kind: Some(RecordKind::Answer), last: None, diff --git a/crates/workshop/server/tests/it/std_real_directory.rs b/crates/workshop/server/tests/it/std_real_directory.rs index 06a37bc8..c0226205 100644 --- a/crates/workshop/server/tests/it/std_real_directory.rs +++ b/crates/workshop/server/tests/it/std_real_directory.rs @@ -1,12 +1,16 @@ //! The `std` file tools over a real directory, mounted with //! `RealBackend::rooted`: the acceptance flow a coding agent follows. List //! the directory, find a word with Grep, read a window of the file, and read -//! the next window from the offset the first window's trailer names. +//! the next window from the offset the first window's trailer names. Then +//! the tools that change files: Write makes a file under a directory that +//! is absent, StrReplace edits it, and Delete removes it, and the real +//! directory holds each change. //! //! The tools come from the std Plugin's dynamic library, loaded from a //! Plugin directory as the server loads it, so each call crosses the C -//! contract and each read goes through the loader's file functions to the -//! real backend. +//! contract and each file request goes through the loader's file functions. +//! No Harness runs here, so the test answers each request itself, with one +//! operation on the real backend. use std::future::Future; use std::pin::pin; @@ -16,9 +20,9 @@ use std::task::{Context, Poll, Waker}; use runvil::effect::{ToolCallOrigin, ToolCaller}; use runvil::harness::{PluginReport, insert_plugin_dir}; -use runvil::plugin::{Pack, PackRegistry, ToolContext}; -use runvil::vfs::{Access, Origin, RealBackend, VfsRef}; -use runvil::{OutputTrust, ToolId, ToolOutput}; +use runvil::plugin::{Pack, PackRegistry, ToolContext, ToolFiles}; +use runvil::vfs::{Access, Origin, RealBackend, VfsError, VfsOp, VfsOutcome, VfsRef}; +use runvil::{BoxFuture, OutputTrust, ToolId, ToolOutput}; use serde_json::{Value, json}; use crate::common::plugin_dir; @@ -109,7 +113,20 @@ fn block_on(future: F) -> F::Output { } } -/// Calls the tool `segment` of `pack` with `args` over `access`. +/// The files of a call that no Harness drives: each request is performed +/// on the access at the first poll of its future, which is ready then. +/// Under the Harness a request is an effect, and a later poll of the call +/// receives its answer. +struct AnsweredAtOnce<'a>(&'a Access); + +impl ToolFiles for AnsweredAtOnce<'_> { + fn request(&self, op: VfsOp) -> BoxFuture<'_, Result> { + Box::pin(async move { self.0.perform(&op) }) + } +} + +/// Calls the tool `segment` of `pack` with `args`, with each file request +/// of the call answered over `access`. #[expect( clippy::expect_used, reason = "test helpers fail by panicking with the invariant named" @@ -117,7 +134,8 @@ fn block_on(future: F) -> F::Output { fn call(pack: &dyn Pack, access: &Access, segment: &str, args: Value) -> ToolOutput { let tool = ToolId::parse(&format!("std/{segment}")).expect("a two-segment tool id parses"); let origin = ToolCallOrigin::new("Test", ToolCaller::Script); - let context = ToolContext::new(&tool, access, &origin); + let files = AnsweredAtOnce(access); + let context = ToolContext::new(&tool, &origin, &files); block_on(pack.call(context, args)).expect("the call succeeds") } @@ -175,3 +193,64 @@ fn list_find_read_a_window_and_read_the_next_one_by_the_trailer_offset() { "the second window holds the line Grep found" ); } + +/// The file the change flow makes, under a directory the seeded one lacks. +const NOTE: &str = "/notes/todo.md"; + +#[test] +fn write_makes_a_file_str_replace_edits_it_and_delete_removes_it() { + let directory = seeded_directory(); + let backend = RealBackend::rooted(directory.path()).expect("the directory mounts"); + let vfs = VfsRef::builder().mount("/", backend).build(); + let access = vfs + .acquire(Origin::new("std real directory")) + .expect("the run acquires the filesystem"); + let (pack, _report) = std_pack(); + let note = directory.path().join("notes").join("todo.md"); + let on_disk = || std::fs::read_to_string(¬e).expect("the real file is read"); + + let written = call( + &*pack, + &access, + "write", + json!({ "path": NOTE, "contents": "alpha\nbeta\n" }), + ); + assert_eq!(written.trust(), OutputTrust::Trusted); + assert_eq!(written.text(), "Wrote 11 bytes."); + assert_eq!( + on_disk(), + "alpha\nbeta\n", + "Write makes the file and the directory on the way to it" + ); + + let edited = call( + &*pack, + &access, + "replace", + json!({ "path": NOTE, "old_string": "beta", "new_string": "BETA" }), + ); + assert_eq!(edited.trust(), OutputTrust::Trusted); + assert_eq!(edited.text(), "Made 1 replacement."); + assert_eq!( + on_disk(), + "alpha\nBETA\n", + "StrReplace changes the one text" + ); + + let read = call(&*pack, &access, "read", json!({ "path": NOTE })); + assert_eq!(read.trust(), OutputTrust::Untrusted); + assert_eq!(read.text(), " 1|alpha\n 2|BETA\n"); + + let deleted = call(&*pack, &access, "rm", json!({ "path": NOTE })); + assert_eq!(deleted.trust(), OutputTrust::Trusted); + assert_eq!(deleted.text(), "Deleted the file."); + assert!(!note.exists(), "Delete removes the file"); + assert!( + directory.path().join("notes").is_dir(), + "the directory Write made stays" + ); + assert!( + directory.path().join(FILE).is_file(), + "the seeded file is left alone" + ); +} diff --git a/deny.toml b/deny.toml index 4ac7e957..eaa32d99 100644 --- a/deny.toml +++ b/deny.toml @@ -14,7 +14,13 @@ version = 2 unmaintained = "workspace" yanked = "deny" # Reviewed advisories consciously accepted go here, each with a rationale. -ignore = [] +ignore = [ + # humantime is marked unmaintained; the advisory reports no vulnerability + # and no safe upgrade. workshop-workspace uses one function of it, + # `format_rfc3339_seconds`, to stamp a time. The entry goes when the + # crate is replaced. + { id = "RUSTSEC-2026-0542", reason = "humantime is unmaintained with no vulnerability reported and no safe upgrade; remove this entry when the crate is replaced" }, +] [licenses] version = 2