diff --git a/AGENTS.md b/AGENTS.md index 79697e108..68c7664b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ Multi-crate Rust workspace for the Engine, the Harness that runs it, and the Rus Seven terms have exactly one meaning each, everywhere in this repository: code comments, docs, rulebooks, and plans. Write them capitalized. -- **Engine**: the `runvil-*` crates. The Engine parses a prompt and steps a run. Whenever a run needs a model reply, a Pack's render of the tools a section offers it, a Pack tool's result, a timer, or a store operation, the Engine emits an effect and waits for its caller to answer it. In production the caller is the Harness. +- **Engine**: the `runvil-*` crates. The Engine parses a prompt and steps a run. Whenever a run needs a model reply, a Pack's render of the tools a section offers it, a Pack tool's result, a timer, or a file operation, the Engine emits an effect and waits for its caller to answer it. In production the caller is the Harness. - **Harness**: the `runvil` crate, which is the one crate a Host depends on, and the `harness-*` crates. The Harness steps the Engine, performs every effect, returns each answer, and records every run through the recorder the Host supplies. Production code runs prompts only through the Harness. - **Host**: an application that runs prompts through the Harness, such as Workshop or Papergate. The Host makes every policy decision. It may also use the Engine's parser and types to read prompts and show events. - **Pack**: code which installs an optional native prelude, an optional Lua prelude, and zero or more tools. diff --git a/Cargo.lock b/Cargo.lock index a5d6c299f..f622c50d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -914,6 +914,7 @@ dependencies = [ "libloading", "runvil", "runvil-plugin", + "serde_json", "tokio", ] @@ -2125,8 +2126,8 @@ dependencies = [ [[package]] name = "runvil-plugin-sys" -version = "0.4.0" -source = "git+https://github.com/cppalliance/runvil-plugin-api?rev=03a9fa37d2fe690d42ee0aa5c7a8289173e8e0f2#03a9fa37d2fe690d42ee0aa5c7a8289173e8e0f2" +version = "0.5.0" +source = "git+https://github.com/cppalliance/runvil-plugin-api?rev=52a4905aca0a9c945a93dc4ee7ea815db85612e4#52a4905aca0a9c945a93dc4ee7ea815db85612e4" [[package]] name = "runvil-prompt" diff --git a/Cargo.toml b/Cargo.toml index 7ea392035..5d5428ef3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,12 +22,12 @@ bytes = "1" runvil-engine = { path = "crates/runvil-engine" } runvil-runtime = { path = "crates/runvil-internal/runtime" } runvil-plugin = { path = "crates/runvil-plugin" } -# The C Plugin contract, version 0.4, from its own repository, pinned by +# The C Plugin contract, version 0.5, from its own repository, pinned by # commit. `version` beside `git` keeps cargo-deny's wildcard check quiet. # The crate sets `links`, so Cargo refuses two commits of the contract in # one build: a Host that also names `runvil-plugin-sdk`, or a Plugin built # on it, pins this same commit. -runvil-plugin-sys = { version = "0.4.0", git = "https://github.com/cppalliance/runvil-plugin-api", rev = "03a9fa37d2fe690d42ee0aa5c7a8289173e8e0f2" } +runvil-plugin-sys = { version = "0.5.0", git = "https://github.com/cppalliance/runvil-plugin-api", rev = "52a4905aca0a9c945a93dc4ee7ea815db85612e4" } runvil-types = { path = "crates/runvil-internal/types" } runvil = { path = "crates/runvil" } harness-runner = { path = "crates/harness-internal/runner" } diff --git a/crates/build-xtask/src/facade_shape-tests.rs b/crates/build-xtask/src/facade_shape-tests.rs index 8bec77011..9df73f95a 100644 --- a/crates/build-xtask/src/facade_shape-tests.rs +++ b/crates/build-xtask/src/facade_shape-tests.rs @@ -186,7 +186,7 @@ fn a_re_export_rooted_outside_the_facade_dependencies_is_rejected() { ), ("pub use std::sync::Arc;", 1), ("pub use ::serde::Serialize;", 1), - ("pub use runvil_lua::VfsOp;", 1), + ("pub use runvil_lua::RosterEntry;", 1), ] { assert_one(text, line, RULE); } diff --git a/crates/harness-internal/plugin-loader/Cargo.toml b/crates/harness-internal/plugin-loader/Cargo.toml index b29d99ab8..1f0eddb2c 100644 --- a/crates/harness-internal/plugin-loader/Cargo.toml +++ b/crates/harness-internal/plugin-loader/Cargo.toml @@ -12,7 +12,7 @@ libloading.workspace = true # The Pack registry the inserts take, `PackId`, which validates a Plugin's # name, and the universal error a failed load is. runvil-plugin.workspace = true -# The C contract every entry point follows, version 0.4, as Rust types. +# The C contract every entry point follows, version 0.5, as Rust types. runvil-plugin-sys.workspace = true # A tool's parameters arrive as JSON text and its arguments leave as JSON # text, because the C contract carries no JSON type. diff --git a/crates/harness-internal/plugin-loader/src/call-tests.rs b/crates/harness-internal/plugin-loader/src/call-tests.rs index 4c0a471d5..6337fb0e8 100644 --- a/crates/harness-internal/plugin-loader/src/call-tests.rs +++ b/crates/harness-internal/plugin-loader/src/call-tests.rs @@ -1,7 +1,9 @@ //! Tests for a tool call through a Plugin's entry point: what each step //! is sent, how a result becomes an output or a failure, when the future -//! is pending, and that RV_TOOL_CANCEL comes once for a call whose last -//! step returned RV_PENDING and for no other call. +//! is pending, when a file request of a step is asked of the call's +//! files and which step is lent its answer, and that RV_TOOL_CANCEL +//! comes once for a call whose last step returned RV_PENDING and for no +//! other call. use std::future::Future; use std::pin::pin; @@ -9,7 +11,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::task::{Context, Poll, Wake, Waker}; -use runvil_plugin::{ErrorCode, OutputTrust}; +use runvil_plugin::{ErrorCode, OutputTrust, VfsError, VfsOp, VfsOutcome}; use serde_json::json; use crate::fakes::*; @@ -223,6 +225,281 @@ fn a_pending_call_that_set_no_state_is_canceled_with_null() { assert!(stateless::CANCELED.load(Ordering::SeqCst).is_null()); } +/// The request every fake below makes: three bytes of one file. +fn three_bytes() -> VfsOp { + VfsOp::ReadRange { + path: "/notes/a.md".to_owned(), + offset: 2, + len: 3, + } +} + +#[test] +fn a_request_goes_to_the_calls_files_at_once_and_the_call_gets_no_step_until_its_outcome_is_there() +{ + pack_fake!( + plug, + { + pub(super) static STEPS: AtomicUsize = AtomicUsize::new(0); + }, + new: |call| { + set_pack_run(call, instance(PACK_TAG)); + add_tool(call, "tool") + }, + step: |call| { + STEPS.fetch_add(1, Ordering::SeqCst); + if tool_call(call).is_null() { + set_tool_call(call, instance(0x300)); + ask_read(call, "/notes/a.md", (2, 3)); + return RV_PENDING; + } + finish(call, RV_RESULT_UNTRUSTED, &format!("{:?}", read_answer(call))) + } + ); + let pack = made(plug::entry); + let call = test_call("tool"); + let files = HeldFiles::default(); + let mut future = pin!(pack.call(context_over(&call, &files), json!({}))); + let count = Arc::new(Wakes::default()); + let waker = Waker::from(Arc::clone(&count)); + let mut cx = Context::from_waker(&waker); + + assert!(future.as_mut().poll(&mut cx).is_pending()); + assert_eq!( + files.asked(), + [three_bytes()], + "the request was made in the poll that sent its step" + ); + assert!(future.as_mut().poll(&mut cx).is_pending()); + assert_eq!( + plug::STEPS.load(Ordering::SeqCst), + 1, + "a call that waits for the outcome of its request gets no step" + ); + assert_eq!( + count.0.load(Ordering::SeqCst), + 0, + "and it does not wake its own task" + ); + + files.answer(0, Ok(VfsOutcome::Bytes(b"llo".to_vec()))); + assert_eq!( + count.0.load(Ordering::SeqCst), + 1, + "the files wake the call's task when the outcome is there" + ); + let Poll::Ready(result) = future.as_mut().poll(&mut cx) else { + panic!("the step that is lent the answer finishes the call"); + }; + assert_eq!( + result.expect("the call succeeds").text(), + format!("Some(Ok({:?}))", b"llo") + ); + assert_eq!(plug::STEPS.load(Ordering::SeqCst), 2); + assert_eq!(files.asked().len(), 1, "the call asked once"); + assert_eq!(plug::CANCELS.load(Ordering::SeqCst), 0); +} + +#[test] +fn a_request_whose_outcome_is_there_at_once_is_answered_in_the_step_of_the_next_poll() { + pack_fake!( + plug, + { + pub(super) static STEPS: AtomicUsize = AtomicUsize::new(0); + }, + new: |call| { + set_pack_run(call, instance(PACK_TAG)); + add_tool(call, "tool") + }, + step: |call| { + STEPS.fetch_add(1, Ordering::SeqCst); + if tool_call(call).is_null() { + set_tool_call(call, instance(0x300)); + ask_read(call, "/notes/a.md", (2, 3)); + return RV_PENDING; + } + finish(call, RV_RESULT_UNTRUSTED, &format!("{:?}", read_answer(call))) + } + ); + let pack = made(plug::entry); + // The files of a `TestCall` answer each request at its first poll. + let call = seeded_call("tool", &[("/notes/a.md", "hello")]); + let mut future = pin!(pack.call(call.context(), json!({}))); + let count = Arc::new(Wakes::default()); + let waker = Waker::from(Arc::clone(&count)); + let mut cx = Context::from_waker(&waker); + + assert!(future.as_mut().poll(&mut cx).is_pending()); + assert_eq!( + plug::STEPS.load(Ordering::SeqCst), + 1, + "a poll sends one step at most" + ); + assert_eq!( + count.0.load(Ordering::SeqCst), + 1, + "the call wakes its own task for the step that is lent the answer" + ); + let Poll::Ready(result) = future.as_mut().poll(&mut cx) else { + panic!("the step that is lent the answer finishes the call"); + }; + assert_eq!( + result.expect("the call succeeds").text(), + format!("Some(Ok({:?}))", b"llo") + ); +} + +#[test] +fn only_the_step_after_a_request_is_lent_an_answer() { + pack_fake!( + plug, + { + /// Whether each step was lent an answer, in order. + pub(super) static LENT: Mutex> = Mutex::new(Vec::new()); + }, + new: |call| { + set_pack_run(call, instance(PACK_TAG)); + add_tool(call, "tool") + }, + step: |call| { + // The call's state counts its steps. + let steps = tool_call(call).addr(); + set_tool_call(call, ptr::without_provenance_mut(steps + 1)); + if let Ok(mut all) = LENT.lock() { + all.push(lent(call).is_some()); + } + match steps { + // The second step asks, and the fourth finishes. + 1 => { + ask_stat(call, "/notes/a.md"); + RV_PENDING + } + 0 | 2 => RV_PENDING, + _ => finish(call, RV_RESULT_TRUSTED, "done"), + } + } + ); + let pack = made(plug::entry); + let call = seeded_call("tool", &[("/notes/a.md", "hello")]); + block_on(pack.call(call.context(), json!({}))).expect("the call succeeds"); + assert_eq!( + *plug::LENT.lock().expect("the lock holds"), + [false, false, true, false], + "the step after the request is lent its answer, and no other step is lent one" + ); +} + +#[test] +fn a_failed_operation_is_lent_as_its_error_and_the_call_goes_on() { + pack_fake!( + plug, + {}, + new: |call| { + set_pack_run(call, instance(PACK_TAG)); + add_tool(call, "tool") + }, + step: |call| { + if tool_call(call).is_null() { + set_tool_call(call, instance(0x300)); + ask_read(call, "/notes/a.md", (2, 3)); + return RV_PENDING; + } + finish(call, RV_RESULT_UNTRUSTED, &format!("{:?}", read_answer(call))) + } + ); + let pack = made(plug::entry); + let call = test_call("tool"); + let files = HeldFiles::default(); + let mut future = pin!(pack.call(context_over(&call, &files), json!({}))); + let mut cx = Context::from_waker(Waker::noop()); + assert!(future.as_mut().poll(&mut cx).is_pending()); + files.answer( + 0, + Err(VfsError::PermissionDenied { + path: "/notes/a.md".to_owned(), + reason: "this run reads nothing under /notes".to_owned(), + }), + ); + let Poll::Ready(result) = future.as_mut().poll(&mut cx) else { + panic!("the step that is lent the answer finishes the call"); + }; + assert_eq!( + result.expect("the call succeeds").text(), + "Some(Err((4, \"this run reads nothing under /notes\")))" + ); +} + +#[test] +fn dropping_a_call_that_waits_for_an_outcome_sends_one_cancel_and_no_step() { + pack_fake!( + plug, + { + pub(super) static STEPS: AtomicUsize = AtomicUsize::new(0); + }, + new: |call| { + set_pack_run(call, instance(PACK_TAG)); + add_tool(call, "tool") + }, + step: |call| { + STEPS.fetch_add(1, Ordering::SeqCst); + set_tool_call(call, instance(0x300)); + ask_read(call, "/notes/a.md", (2, 3)); + RV_PENDING + } + ); + let pack = made(plug::entry); + let call = test_call("tool"); + let files = HeldFiles::default(); + let mut future = Box::pin(pack.call(context_over(&call, &files), json!({}))); + let mut cx = Context::from_waker(Waker::noop()); + assert!(future.as_mut().poll(&mut cx).is_pending()); + assert_eq!(files.asked(), [three_bytes()]); + drop(future); + assert_eq!(plug::CANCELS.load(Ordering::SeqCst), 1); + assert_eq!(plug::CANCELED.load(Ordering::SeqCst), instance(0x300)); + files.answer(0, Ok(VfsOutcome::Bytes(b"llo".to_vec()))); + assert_eq!( + plug::STEPS.load(Ordering::SeqCst), + 1, + "an abandoned call gets no step for an outcome that comes late" + ); +} + +#[test] +fn the_request_of_a_step_that_ends_its_call_is_not_asked_of_the_files() { + pack_fake!( + plug, + {}, + new: |call| { + set_pack_run(call, instance(PACK_TAG)); + add_tool(call, "tool") + }, + step: |call| { + ask_read(call, "/notes/a.md", (2, 3)); + // The arguments are one JSON number, the status to return. + text(step_args(call).arguments).parse().unwrap_or(RV_OK) + } + ); + let pack = made(plug::entry); + for (status, message) in [ + ( + RV_OK, + "the tool's Plugin answered RV_TOOL_CALL with RV_OK without finishing the call", + ), + ( + ODD_STATUS, + "the tool's Plugin answered RV_TOOL_CALL with status 77", + ), + ] { + let call = test_call("tool"); + let files = HeldFiles::default(); + let failure = block_on(pack.call(context_over(&call, &files), json!(status))) + .expect_err("the call fails"); + assert_eq!(failure.message, message); + assert_eq!(files.asked(), [], "status {status}"); + } +} + /// Defines a test whose fake step runs `$step`, which returns the step's /// status. The call must fail as a `ToolOther` error whose message is /// `$message`, and must get `$cancels` RV_TOOL_CANCEL calls. @@ -282,6 +559,39 @@ failed_call_test!( } ); +failed_call_test!( + a_second_request_in_one_step_fails_the_call_and_is_canceled, + misuse_reason!("a step made a second file request"), + 1, + |call| { + ask_stat(call, "/a"); + ask_stat(call, "/a"); + RV_PENDING + } +); + +failed_call_test!( + a_finish_after_a_request_in_one_step_fails_the_call, + misuse_reason!("a step called finish after it made a file request"), + 0, + |call| { + ask_stat(call, "/a"); + finish(call, RV_RESULT_TRUSTED, "done"); + RV_OK + } +); + +failed_call_test!( + a_request_after_finish_in_one_step_fails_the_call, + misuse_reason!("a step made a file request after it called finish"), + 0, + |call| { + finish(call, RV_RESULT_TRUSTED, "done"); + ask_stat(call, "/a"); + RV_OK + } +); + failed_call_test!( a_misuse_fails_the_call_even_when_a_good_result_follows, misuse_reason!("the result's kind is not one of the RV_RESULT constants"), diff --git a/crates/harness-internal/plugin-loader/src/call.rs b/crates/harness-internal/plugin-loader/src/call.rs index 76a684bb5..95899c32b 100644 --- a/crates/harness-internal/plugin-loader/src/call.rs +++ b/crates/harness-internal/plugin-loader/src/call.rs @@ -1,7 +1,7 @@ //! One tool call of a Pack instance that a Plugin's entry point made: the -//! future that sends RV_TOOL_CALL once each time it is polled, the -//! function an `RvToolCallArgs` carries to end the call, and the sender -//! of RV_TOOL_CANCEL. +//! future that sends its RV_TOOL_CALL steps and asks the call's files +//! for each request a step makes, the function an `RvToolCallArgs` +//! carries to end the call, and the sender of RV_TOOL_CANCEL. use std::ffi::c_void; use std::future::Future; @@ -9,7 +9,10 @@ use std::pin::Pin; use std::ptr; use std::task::{Context, Poll}; -use runvil_plugin::{Access, Error, ErrorCode, ToolOutput, make_tool_error}; +use runvil_plugin::{ + BoxFuture, Error, ErrorCode, ToolFiles, ToolOutput, VfsError, VfsOp, VfsOutcome, + make_tool_error, +}; use runvil_plugin_sys::{ RV_MAX_RESULT, RV_OK, RV_PENDING, RV_RESULT_BACKEND, RV_RESULT_INVALID_ARGUMENTS, RV_RESULT_OTHER, RV_RESULT_TRANSPORT, RV_RESULT_TRUSTED, RV_RESULT_UNTRUSTED, RV_TOOL_CALL, @@ -17,10 +20,13 @@ use runvil_plugin_sys::{ }; use serde_json::Value; -use crate::file::{file_list, file_read, file_stat}; +use crate::file::{ + Answer, FINISH_AFTER_REQUEST, Wanted, file_append, file_copy, file_list, file_mkdir, file_read, + file_remove, file_rename, file_stat, file_write, +}; use crate::pack::EntryPack; use crate::session::{ - self, Field, Session, Shape, TOOL_CALL, answer, field, rv_str, shape, size_field, + self, Field, Session, Shape, TOOL_CALL, Verb, answer, field, rv_str, shape, size_field, }; #[cfg(test)] @@ -30,8 +36,13 @@ mod tests; const RESULT: Shape = shape!("finish", "result", "RvToolResult"); const TEXT: Field = field!("the result's text", 16_777_216, RV_MAX_RESULT); -/// One tool call in flight. Each poll sends one RV_TOOL_CALL step. A step -/// that returns RV_PENDING wakes the future's own task and leaves it +/// One tool call in flight. A poll sends at most one RV_TOOL_CALL step. +/// +/// When a step made a file request, the future asks the call's files for +/// it and then waits for the outcome without waking itself: the files +/// wake its task when the outcome is there, and the poll after that +/// sends the step that is lent the answer. A step that returns +/// RV_PENDING with no request wakes the future's own task and leaves it /// pending, so the Host's executor gives the run's other work a turn and /// then polls again. Dropping the future while its last step returned /// RV_PENDING sends RV_TOOL_CANCEL, so the Plugin frees the call's state. @@ -40,8 +51,8 @@ const TEXT: Field = field!("the result's text", 16_777_216, RV_MAX_RESULT); /// every one of its calls has ended. pub(crate) struct Steps<'a> { pack: &'a EntryPack, - /// The call's files, which each step's file functions read. - access: &'a Access, + /// The call's files, which each request of a step is asked of. + files: &'a dyn ToolFiles, /// The tool's name within its Pack. tool: String, /// The JSON text of the call's arguments. @@ -54,36 +65,63 @@ pub(crate) struct Steps<'a> { pending: bool, /// Whether the call has ended. ended: bool, + /// The request the last step made, while the call waits for its + /// outcome. + request: Option>, + /// The answer to the request the last step made, which the next step + /// is lent. + answer: Option, +} + +/// A request the call's files were asked for: what it asks for, and the +/// future of its outcome. +struct Asked<'a> { + wanted: Wanted, + outcome: BoxFuture<'a, Result>, +} + +/// What one step left of its call. +enum Stepped { + /// The call ended, with this result. + Ended(Result), + /// The step returned RV_PENDING after it made this file request. + Asked(VfsOp), + /// The step returned RV_PENDING and made no request. + Waits, } // SAFETY: the `# Threads and lifetime` section of `runvil-plugin-sys` lets // the steps of one tool call, and its RV_TOOL_CANCEL, come on different // threads, and the loader never reads through `state`. Every other field -// is `Send`, which the assertion below checks for the two borrows. +// is `Send`, which the assertion below checks for the two borrows, the +// request, and the answer. unsafe impl Send for Steps<'_> {} const _: () = { const fn is_send() {} - is_send::<(&EntryPack, &Access)>(); + is_send::<(&EntryPack, &dyn ToolFiles, Asked<'_>, Answer)>(); }; impl<'a> Steps<'a> { - /// A call of the tool named `tool` on `pack` with `arguments`, over - /// the files `access` reaches. Nothing is sent until the first poll. + /// A call of the tool named `tool` on `pack` with `arguments`, whose + /// file requests are asked of `files`. Nothing is sent until the + /// first poll. pub(crate) fn new( pack: &'a EntryPack, tool: &str, arguments: &Value, - access: &'a Access, + files: &'a dyn ToolFiles, ) -> Steps<'a> { Steps { pack, - access, + files, tool: tool.to_owned(), arguments: arguments.to_string(), state: ptr::null_mut(), pending: false, ended: false, + request: None, + answer: None, } } @@ -95,11 +133,16 @@ impl<'a> Steps<'a> { } } - /// Sends one RV_TOOL_CALL step, and returns the call's result once it - /// has ended. - fn step(&mut self) -> Option> { + /// Sends one RV_TOOL_CALL step, which is lent the answer to the + /// request the last step made, and returns what the step left of the + /// call. The request of a step that ended the call is not performed. + fn step(&mut self) -> Stepped { let mut call = self.new_call(RV_TOOL_CALL); - let (status, mut session) = session::open_step(self.access, |ctx| { + // The answer and what it points to stay valid until this function + // returns, which is after the step has. + let answer = self.answer.take(); + let lent = answer.as_ref().map(Answer::raw); + let (status, mut session) = session::open(Verb::ToolCall, |ctx| { let args = RvToolCallArgs { size: size_field::(), reserved: 0, @@ -110,20 +153,49 @@ impl<'a> Steps<'a> { file_stat, file_list, file_read, + answer: lent.as_ref().map_or(ptr::null(), ptr::from_ref), + file_write, + file_append, + file_mkdir, + file_remove, + file_rename, + file_copy, }; call.args = (&raw const args).cast(); // SAFETY: `call` carries this call's own pointers and a live - // `RvToolCallArgs`, and no other step of the call runs, - // because a step needs `&mut self`. + // `RvToolCallArgs`, whose `answer` is NULL or a live + // `RvFileAnswer`, and no other step of the call runs, because + // a step needs `&mut self`. unsafe { self.pack.send(&mut call) } }); self.state = call.tool_call; self.pending = status == RV_PENDING; - let outcome = judge(status, &mut session); - if outcome.is_some() { - self.end(); + let request = session.request.take(); + match (judge(status, &mut session), request) { + (Some(result), _) => { + self.end(); + Stepped::Ended(result) + } + (None, Some(op)) => Stepped::Asked(op), + (None, None) => Stepped::Waits, + } + } + + /// Asks the call's files for `op`, the request the last step made, + /// and polls the request at once, because a request is made at the + /// first poll of its future. A request whose outcome is there + /// already wakes the task for the step that is lent its answer. + fn ask(&mut self, op: VfsOp, cx: &mut Context<'_>) { + let files: &'a dyn ToolFiles = self.files; + let wanted = Wanted::by(&op); + let mut outcome = files.request(op); + match outcome.as_mut().poll(cx) { + Poll::Ready(outcome) => { + self.answer = Some(Answer::to(wanted, outcome)); + cx.waker().wake_by_ref(); + } + Poll::Pending => self.request = Some(Asked { wanted, outcome }), } - outcome } /// Ends the call: sends RV_TOOL_CANCEL when its last step returned @@ -150,13 +222,30 @@ impl Future for Steps<'_> { "the tool call was polled after it ended".to_owned(), ))); } - if let Some(result) = this.step() { - return Poll::Ready(result); + if let Some(asked) = this.request.as_mut() { + // The contract sends a call no step while its request waits + // for an answer. The call's files wake this task when the + // outcome is there, so the future wakes nothing itself. + let Poll::Ready(outcome) = asked.outcome.as_mut().poll(cx) else { + return Poll::Pending; + }; + this.answer = Some(Answer::to(asked.wanted, outcome)); + this.request = None; + } + match this.step() { + Stepped::Ended(result) => Poll::Ready(result), + Stepped::Asked(op) => { + this.ask(op, cx); + Poll::Pending + } + Stepped::Waits => { + // The contract has no wake function: a pending call that + // asked for nothing gets its next step as soon as the + // executor polls again. + cx.waker().wake_by_ref(); + Poll::Pending + } } - // The contract has no wake function: a pending call gets its next - // step as soon as the executor polls again. - cx.waker().wake_by_ref(); - Poll::Pending } } @@ -207,6 +296,9 @@ pub(crate) unsafe extern "C" fn finish(ctx: *mut c_void, result: *const RvToolRe if session.result.is_some() { return Err("finish was called twice"); } + if session.request.is_some() { + return Err(FINISH_AFTER_REQUEST); + } // SAFETY: the contract makes `result` point at a readable // `RvToolResult` during this call. let result = unsafe { RESULT.read(result) }?; diff --git a/crates/harness-internal/plugin-loader/src/fakes.rs b/crates/harness-internal/plugin-loader/src/fakes.rs index 32dd431df..a7bb979f2 100644 --- a/crates/harness-internal/plugin-loader/src/fakes.rs +++ b/crates/harness-internal/plugin-loader/src/fakes.rs @@ -4,24 +4,27 @@ //! `fake!` or `pack_fake!`, so its statics belong to it alone. use std::ffi::c_void; -use std::future::Future; +use std::future::{Future, poll_fn}; use std::pin::pin; use std::ptr; use std::slice; -use std::sync::Arc; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::task::{Context, Poll, Waker}; use runvil_plugin::testing::TestCall; -use runvil_plugin::{Error, Pack, PackRegistry, ToolId, ToolOutput}; +use runvil_plugin::{ + BoxFuture, Error, Pack, PackRegistry, ToolContext, ToolFiles, ToolId, ToolOutput, VfsError, + VfsOp, VfsOutcome, +}; pub(crate) use runvil_plugin_sys::{ - PluginEntry, RV_ERR_MISUSE, RV_ERR_REFUSED, RV_ERR_UNSUPPORTED, RV_FILE_ERROR_IS_A_DIRECTORY, - RV_FILE_ERROR_NOT_A_DIRECTORY, RV_FILE_ERROR_NOT_FOUND, RV_FILE_ERROR_OTHER, - RV_FILE_TYPE_DIRECTORY, RV_FILE_TYPE_FILE, RV_OK, RV_PACK_DELETE, RV_PACK_NEW, RV_PENDING, - RV_PLUGIN_LOAD, RV_PLUGIN_UNLOAD, RV_RESULT_BACKEND, RV_RESULT_INVALID_ARGUMENTS, - RV_RESULT_OTHER, RV_RESULT_TRANSPORT, RV_RESULT_TRUSTED, RV_RESULT_UNTRUSTED, RV_TOOL_CALL, - RV_TOOL_CANCEL, RvBytes, RvCall, RvDirEntries, RvFileError, RvHost, RvLoadArgs, RvPackInfo, - RvPackNewArgs, RvPluginInfo, RvResultKind, RvStat, RvStatus, RvStr, RvToolCallArgs, RvToolInfo, - RvToolResult, RvVerb, + PluginEntry, RV_ERR_FILE, RV_ERR_MISUSE, RV_ERR_REFUSED, RV_ERR_UNSUPPORTED, + RV_FILE_ERROR_IS_A_DIRECTORY, RV_FILE_ERROR_NOT_A_DIRECTORY, RV_FILE_ERROR_NOT_FOUND, + RV_FILE_ERROR_OTHER, RV_FILE_TYPE_DIRECTORY, RV_FILE_TYPE_FILE, RV_OK, RV_PACK_DELETE, + RV_PACK_NEW, RV_PENDING, RV_PLUGIN_LOAD, RV_PLUGIN_UNLOAD, RV_RESULT_BACKEND, + RV_RESULT_INVALID_ARGUMENTS, RV_RESULT_OTHER, RV_RESULT_TRANSPORT, RV_RESULT_TRUSTED, + RV_RESULT_UNTRUSTED, RV_TOOL_CALL, RV_TOOL_CANCEL, RV_TOOL_CHANGES_FILES, RvBytes, RvCall, + RvFileAnswer, RvHost, RvLoadArgs, RvPackInfo, RvPackNewArgs, RvPluginInfo, RvResultKind, + RvStatus, RvStr, RvToolCallArgs, RvToolInfo, RvToolResult, RvVerb, }; use serde_json::Value; @@ -260,7 +263,7 @@ pub(crate) fn size_u32() -> u32 { } /// Whether `call` carries the fields the contract gives a call that sends -/// `verb` in version 0.4, and its table says the contract's version. Each +/// `verb` in version 0.5, and its table says the contract's version. Each /// verb carries the Plugin's pointers the contract names for it and no /// others, and its own arguments struct, with a `ctx`, or no arguments. pub(crate) fn contract_holds(call: *mut RvCall, verb: RvVerb) -> bool { @@ -296,7 +299,7 @@ pub(crate) fn contract_holds(call: *mut RvCall, verb: RvVerb) -> bool { && args_hold && table.size == size_u32::() && table.api_major == 0 - && table.api_minor == 4 + && table.api_minor == 5 && table.reserved == 0 } @@ -486,12 +489,12 @@ pub(crate) fn insert_pack(call: *mut RvCall, name: &str, description: &str) -> R insert_pack_info(call, &pack_info(rv_str(name), rv_str(description))) } -/// A well-formed `RvToolInfo` for the tool `name`, with no wire name, a -/// one-line description, and [`NO_PARAMETERS`]. +/// A well-formed `RvToolInfo` for the tool `name`, with no flag, no wire +/// name, a one-line description, and [`NO_PARAMETERS`]. pub(crate) fn tool_info(name: &str) -> RvToolInfo { RvToolInfo { size: size_u32::(), - reserved: 0, + flags: 0, name: rv_str(name), wire_name: rv_str(""), description: rv_str("A fake tool."), @@ -556,140 +559,154 @@ pub(crate) fn finish(call: *mut RvCall, kind: RvResultKind, text: &str) -> RvSta finish_result(call, &result(kind, rv_str(text))) } -/// An `RvStat` whose every field is 9, so a test sees whether a file -/// function wrote it. -pub(crate) fn unset_stat() -> RvStat { - RvStat { - file_type: 9, - reserved: 9, - len: 9, +/// A run of bytes for the C side, borrowed from `bytes`. +pub(crate) fn rv_bytes(bytes: &[u8]) -> RvBytes { + RvBytes { + ptr: bytes.as_ptr(), + len: bytes.len(), } } -/// An `RvFileError` of kind 9 that says `untouched`, so a test sees -/// whether a file function wrote it. -pub(crate) fn unset_error() -> RvFileError { - RvFileError { - kind: 9, - reserved: 9, - text: rv_str("untouched"), - } +/// Calls the step's `file_stat` with `as_ctx`, as a Plugin would or as it +/// must not. +pub(crate) fn ask_stat_as(call: *mut RvCall, as_ctx: *mut c_void, path: RvStr) -> RvStatus { + // SAFETY: the function refuses a `ctx` that is not the open step's + // and a path it cannot read before it reads through either. + unsafe { (step_args(call).file_stat)(as_ctx, path) } } -/// An `RvBytes` with no pointer and a length of 9, so a test sees whether -/// a file function wrote it. -pub(crate) fn unset_bytes() -> RvBytes { - RvBytes { - ptr: ptr::null(), - len: 9, - } +/// Asks through the step's `file_stat` what `path` names. +pub(crate) fn ask_stat(call: *mut RvCall, path: &str) -> RvStatus { + ask_stat_as(call, step_args(call).ctx, rv_str(path)) } -/// An `RvDirEntries` with no pointer and a length of 9, so a test sees -/// whether a file function wrote it. -pub(crate) fn unset_entries() -> RvDirEntries { - RvDirEntries { - ptr: ptr::null(), - len: 9, - } +/// Asks through the step's `file_list` for the entries of `path`. +pub(crate) fn ask_list(call: *mut RvCall, path: &str) -> RvStatus { + let args = step_args(call); + // SAFETY: the open step's own `ctx`, and a path that outlives the + // call. + unsafe { (args.file_list)(args.ctx, rv_str(path)) } } -/// Calls the step's `file_stat` with `as_ctx`, as a Plugin would or as it -/// must not. -pub(crate) fn stat_as( - call: *mut RvCall, - as_ctx: *mut c_void, - path: RvStr, - stat: *mut RvStat, - error: *mut RvFileError, -) -> RvStatus { - // SAFETY: the function refuses a `ctx` that is not the open step's, a - // path it cannot read, and a pointer that is NULL or misaligned, - // before it reads or writes through any of them. Every other pointer - // a test hands over is a live struct of its own. - unsafe { (step_args(call).file_stat)(as_ctx, path, stat, error) } +/// Asks through the step's `file_read` for the bytes of `path` in the +/// range `offset` and `len`. +pub(crate) fn ask_read(call: *mut RvCall, path: &str, range: (u64, u64)) -> RvStatus { + let args = step_args(call); + // SAFETY: as in `ask_list`. + unsafe { (args.file_read)(args.ctx, rv_str(path), range.0, range.1) } } -/// Calls the step's `file_list` with `as_ctx`, as `stat_as` calls -/// `file_stat`. -pub(crate) fn list_as( - call: *mut RvCall, - as_ctx: *mut c_void, - path: RvStr, - entries: *mut RvDirEntries, - error: *mut RvFileError, -) -> RvStatus { - // SAFETY: as in `stat_as`. - unsafe { (step_args(call).file_list)(as_ctx, path, entries, error) } +/// Asks through the step's `file_write` to write `bytes` to `path`. +pub(crate) fn ask_write(call: *mut RvCall, path: &str, bytes: RvBytes) -> RvStatus { + let args = step_args(call); + // SAFETY: as in `ask_list`, and the function refuses bytes it cannot + // read before it reads through them. + unsafe { (args.file_write)(args.ctx, rv_str(path), bytes) } } -/// Calls the step's `file_read` with `as_ctx`, as `stat_as` calls -/// `file_stat`. -pub(crate) fn read_as( - call: *mut RvCall, - as_ctx: *mut c_void, - path: RvStr, - range: (u64, u64), - bytes: *mut RvBytes, - error: *mut RvFileError, -) -> RvStatus { - // SAFETY: as in `stat_as`. - unsafe { (step_args(call).file_read)(as_ctx, path, range.0, range.1, bytes, error) } -} - -/// What a file function of a step answered: what it handed back, or the -/// kind and the text of its failure. -pub(crate) type Answer = Result; - -/// What the step's `file_stat` says of `path`: the file type and the -/// length, or the kind and the text of the failure. -pub(crate) fn stat_of(call: *mut RvCall, path: &str) -> Answer<(u32, u64)> { - let (mut stat, mut error) = (unset_stat(), unset_error()); - let ctx = step_args(call).ctx; - match stat_as(call, ctx, rv_str(path), &raw mut stat, &raw mut error) { - RV_OK => Ok((stat.file_type, stat.len)), - _ => Err((error.kind, text(error.text))), - } +/// Asks through the step's `file_append` to add `bytes` to `path`. +pub(crate) fn ask_append(call: *mut RvCall, path: &str, bytes: RvBytes) -> RvStatus { + let args = step_args(call); + // SAFETY: as in `ask_write`. + unsafe { (args.file_append)(args.ctx, rv_str(path), bytes) } +} + +/// Asks through the step's `file_mkdir` to make the directory `path`. +pub(crate) fn ask_mkdir(call: *mut RvCall, path: &str, recursive: u32) -> RvStatus { + let args = step_args(call); + // SAFETY: as in `ask_list`. + unsafe { (args.file_mkdir)(args.ctx, rv_str(path), recursive) } +} + +/// Asks through the step's `file_remove` to remove `path`. +pub(crate) fn ask_remove(call: *mut RvCall, path: &str, recursive: u32) -> RvStatus { + let args = step_args(call); + // SAFETY: as in `ask_list`. + unsafe { (args.file_remove)(args.ctx, rv_str(path), recursive) } +} + +/// Asks through the step's `file_rename` to move `from` to `to`. +pub(crate) fn ask_rename(call: *mut RvCall, from: RvStr, to: RvStr) -> RvStatus { + let args = step_args(call); + // SAFETY: the open step's own `ctx`, and the function refuses a path + // it cannot read before it reads through it. + unsafe { (args.file_rename)(args.ctx, from, to) } +} + +/// Asks through the step's `file_copy` to copy `from` to `to`. +pub(crate) fn ask_copy(call: *mut RvCall, from: RvStr, to: RvStr) -> RvStatus { + let args = step_args(call); + // SAFETY: as in `ask_rename`. + unsafe { (args.file_copy)(args.ctx, from, to) } +} + +/// What the answer a step is lent holds for the file function that +/// asked: what the function asked for, or the kind and the text of the +/// failure. `None` for a step that is lent no answer. +pub(crate) type Answered = Option>; + +/// The answer the step is lent, or `None` when the step follows no +/// request. +pub(crate) fn lent<'call>(call: *mut RvCall) -> Option<&'call RvFileAnswer> { + // SAFETY: the loader keeps the answer of a step valid until the step + // returns, which is as long as a fake holds this reference. + unsafe { step_args(call).answer.as_ref() } +} + +/// What `held` reads out of the answer the step is lent, or the answer's +/// failure. +fn answered(call: *mut RvCall, held: impl FnOnce(&RvFileAnswer) -> T) -> Answered { + let answer = lent(call)?; + Some(match answer.status { + RV_OK => Ok(held(answer)), + _ => Err((answer.error.kind, text(answer.error.text))), + }) +} + +/// The answer to a `file_stat`: the file type and the length. +pub(crate) fn stat_answer(call: *mut RvCall) -> Answered<(u32, u64)> { + answered(call, |answer| (answer.stat.file_type, answer.stat.len)) } -/// What the step's `file_list` says of `path`: each entry's name, file -/// type, and length, or the kind and the text of the failure. -pub(crate) fn list_of(call: *mut RvCall, path: &str) -> Answer> { - let (mut entries, mut error) = (unset_entries(), unset_error()); - let ctx = step_args(call).ctx; - match list_as(call, ctx, rv_str(path), &raw mut entries, &raw mut error) { - RV_OK if entries.len == 0 => Ok(Vec::new()), - RV_OK => { - // SAFETY: the loader keeps the entries and their names valid - // until the step's next file function. - let listed = unsafe { slice::from_raw_parts(entries.ptr, entries.len) }; - Ok(listed - .iter() - .map(|entry| (text(entry.name), entry.stat.file_type, entry.stat.len)) - .collect()) +/// The answer to a `file_list`: each entry's name, file type, and length. +pub(crate) fn list_answer(call: *mut RvCall) -> Answered> { + answered(call, |answer| { + if answer.entries.len == 0 { + return Vec::new(); } - _ => Err((error.kind, text(error.text))), - } + // SAFETY: the loader keeps the entries and their names valid + // until the step returns. + let listed = unsafe { slice::from_raw_parts(answer.entries.ptr, answer.entries.len) }; + listed + .iter() + .map(|entry| (text(entry.name), entry.stat.file_type, entry.stat.len)) + .collect() + }) +} + +/// The answer to a `file_read`: the bytes. +pub(crate) fn read_answer(call: *mut RvCall) -> Answered> { + answered(call, |answer| { + if answer.bytes.len == 0 { + return Vec::new(); + } + // SAFETY: the loader keeps the bytes valid until the step + // returns. + unsafe { slice::from_raw_parts(answer.bytes.ptr, answer.bytes.len) }.to_vec() + }) +} + +/// The answer to a file function that changes the files: nothing but +/// whether the change was made. +pub(crate) fn change_answer(call: *mut RvCall) -> Answered<()> { + answered(call, |_answer| ()) } -/// What the step's `file_read` hands back for `path` and the range -/// `offset` and `len`, or the kind and the text of the failure. -pub(crate) fn read_of(call: *mut RvCall, path: &str, range: (u64, u64)) -> Answer> { - let (mut bytes, mut error) = (unset_bytes(), unset_error()); - let ctx = step_args(call).ctx; - match read_as( - call, - ctx, - rv_str(path), - range, - &raw mut bytes, - &raw mut error, - ) { - RV_OK if bytes.len == 0 => Ok(Vec::new()), - // SAFETY: the loader keeps the bytes valid until the step's next - // file function. - RV_OK => Ok(unsafe { slice::from_raw_parts(bytes.ptr, bytes.len) }.to_vec()), - _ => Err((error.kind, text(error.text))), +/// The operation that writes `text` to `path`. +pub(crate) fn write_op(path: &str, text: &str) -> VfsOp { + VfsOp::Write { + path: path.to_owned(), + contents: text.as_bytes().to_vec(), } } @@ -698,10 +715,94 @@ pub(crate) fn read_of(call: *mut RvCall, path: &str, range: (u64, u64)) -> Answe pub(crate) fn seeded_call(name: &str, files: &[(&str, &str)]) -> TestCall { let call = test_call(name); for (path, text) in files { - call.context() - .access() - .write(path, text.as_bytes()) + block_on(call.context().files().request(write_op(path, text))) .expect("a seeded file is written"); } call } + +/// The context `call` lends, with `files` in place of the call's own +/// files. +pub(crate) fn context_over<'a>(call: &'a TestCall, files: &'a dyn ToolFiles) -> ToolContext<'a> { + let lent = call.context(); + ToolContext::new(lent.tool(), lent.origin(), files) +} + +/// The whole text of the file at `path` in the filesystem of `call`. +pub(crate) fn text_at(call: &TestCall, path: &str) -> Result { + let read = VfsOp::Read { + path: path.to_owned(), + start: None, + end: None, + }; + match block_on(call.context().files().request(read))? { + VfsOutcome::Text(text) => Ok(text), + other => panic!("a read answered {other:?}"), + } +} + +/// Files that take each request and hold it until a test answers it. +/// They take a request at the first poll of its future, which is when a +/// request joins its call's queue at the Harness. +#[derive(Default)] +pub(crate) struct HeldFiles { + held: Mutex, +} + +/// The requests a [`HeldFiles`] took. +#[derive(Default)] +struct Held { + /// The operations asked for, in the order the requests were made. + asked: Vec, + /// The outcome a test gave each request, until its future takes it. + outcomes: Vec>>, + /// The waker of the last poll of each request's future. + wakers: Vec>, +} + +impl HeldFiles { + /// The requests, locked. + fn held(&self) -> MutexGuard<'_, Held> { + self.held.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// The operations asked for so far, in the order the requests were + /// made. + pub(crate) fn asked(&self) -> Vec { + self.held().asked.clone() + } + + /// Answers the request numbered `index`, counted from 0 in the order + /// the requests were made, with `outcome`, and wakes its future. + pub(crate) fn answer(&self, index: usize, outcome: Result) { + let waker = { + let mut held = self.held(); + held.outcomes[index] = Some(outcome); + held.wakers[index].take() + }; + if let Some(waker) = waker { + waker.wake(); + } + } +} + +impl ToolFiles for HeldFiles { + fn request(&self, op: VfsOp) -> BoxFuture<'_, Result> { + let mut unsent = Some(op); + let mut index = 0; + Box::pin(poll_fn(move |cx| { + let mut held = self.held(); + if let Some(op) = unsent.take() { + index = held.asked.len(); + held.asked.push(op); + held.outcomes.push(None); + held.wakers.push(None); + } + if let Some(outcome) = held.outcomes[index].take() { + return Poll::Ready(outcome); + } + held.wakers[index] = Some(cx.waker().clone()); + Poll::Pending + })) + } +} diff --git a/crates/harness-internal/plugin-loader/src/file-tests.rs b/crates/harness-internal/plugin-loader/src/file-tests.rs index b91319f0c..7aaeca997 100644 --- a/crates/harness-internal/plugin-loader/src/file-tests.rs +++ b/crates/harness-internal/plugin-loader/src/file-tests.rs @@ -1,24 +1,701 @@ -//! Tests for the file functions of a tool call's step: what each hands -//! back from the call's access, how a `VfsError` reads to a Plugin, that a -//! failed file function leaves the call running, and what each is refused -//! for as a misuse. +//! Tests for the file functions of a tool call's step: the operation each +//! takes as its step's request, what each is refused for as a misuse, +//! what an answer lends the step after a request, how a `VfsError` reads +//! to a Plugin, and whole calls that ask the files of a call. +use std::ffi::c_void; use std::io; +use std::ptr; +use std::slice; use std::sync::atomic::Ordering; -use runvil_plugin::{FileType, PathReason, Stat, VfsError}; +use runvil_plugin::testing::TestCall; +use runvil_plugin::{Entry, FileType, PathReason, Stat, VfsError, VfsOp, VfsOutcome}; use runvil_plugin_sys::{ - RV_FILE_ERROR_CONFLICT, RV_FILE_ERROR_DENIED, RV_FILE_ERROR_IO, RV_FILE_TYPE_OTHER, - RV_FILE_TYPE_SYMLINK, + RV_FILE_ERROR_ALREADY_EXISTS, RV_FILE_ERROR_DENIED, RV_FILE_ERROR_IO, RV_FILE_ERROR_NOT_EMPTY, + RV_FILE_TYPE_OTHER, RV_FILE_TYPE_SYMLINK, }; use serde_json::json; -use super::{Lent, contained, describe, raw_stat}; +use super::{ + Answer, Wanted, describe, file_append, file_copy, file_list, file_mkdir, file_read, + file_remove, file_rename, file_stat, file_write, raw_stat, +}; use crate::fakes::*; +use crate::session::{self, Verb}; -/// The files every call below reads. +/// The files every whole call below works on. const FILES: &[(&str, &str)] = &[("/notes/a.md", "hello"), ("/notes/b.md", "")]; +/// Runs `step` as one RV_TOOL_CALL step, and returns what it returned +/// beside the request the loader took and the misuse it recorded. +fn in_step(step: impl FnOnce(*mut c_void) -> T) -> (T, Option, Option<&'static str>) { + let mut out = None; + let (_status, session) = session::open(Verb::ToolCall, |ctx| { + out = Some(step(ctx)); + RV_OK + }); + (out.expect("the step ran"), session.request, session.misuse) +} + +/// A path as the operations below hold it. +fn path(path: &str) -> String { + path.to_owned() +} + +#[test] +fn each_file_function_that_reads_takes_its_request_as_an_operation() { + // SAFETY: the open step's `ctx` and a live path. + let stat = in_step(|ctx| unsafe { file_stat(ctx, rv_str("/notes/a.md")) }); + let asked = VfsOp::Stat { + path: path("/notes/a.md"), + }; + assert_eq!(stat, (RV_OK, Some(asked), None)); + // SAFETY: as above. + let list = in_step(|ctx| unsafe { file_list(ctx, rv_str("notes")) }); + let asked = VfsOp::List { + path: path("notes"), + }; + assert_eq!(list, (RV_OK, Some(asked), None)); + // SAFETY: as above. + let read = in_step(|ctx| unsafe { file_read(ctx, rv_str("/notes/a.md"), 1, 3) }); + let asked = VfsOp::ReadRange { + path: path("/notes/a.md"), + offset: 1, + len: 3, + }; + assert_eq!(read, (RV_OK, Some(asked), None)); +} + +#[test] +fn each_file_function_that_changes_files_takes_its_request_as_an_operation() { + let bytes = rv_bytes(b"text"); + // SAFETY: the open step's `ctx`, a live path, and live bytes. + let write = in_step(|ctx| unsafe { file_write(ctx, rv_str("/a"), bytes) }); + let asked = VfsOp::Write { + path: path("/a"), + contents: b"text".to_vec(), + }; + assert_eq!(write, (RV_OK, Some(asked), None)); + // SAFETY: as above. + let append = in_step(|ctx| unsafe { file_append(ctx, rv_str("/a"), bytes) }); + let asked = VfsOp::Append { + path: path("/a"), + contents: b"text".to_vec(), + }; + assert_eq!(append, (RV_OK, Some(asked), None)); + // SAFETY: the open step's `ctx` and a live path. + let mkdir = in_step(|ctx| unsafe { file_mkdir(ctx, rv_str("/a"), 1) }); + let asked = VfsOp::Mkdir { + path: path("/a"), + recursive: true, + }; + assert_eq!(mkdir, (RV_OK, Some(asked), None)); + // SAFETY: as above. + let remove = in_step(|ctx| unsafe { file_remove(ctx, rv_str("/a"), 0) }); + let asked = VfsOp::Remove { + path: path("/a"), + recursive: false, + }; + assert_eq!(remove, (RV_OK, Some(asked), None)); + // SAFETY: the open step's `ctx` and two live paths. + let rename = in_step(|ctx| unsafe { file_rename(ctx, rv_str("/a"), rv_str("/b")) }); + let asked = VfsOp::Rename { + from: path("/a"), + to: path("/b"), + }; + assert_eq!(rename, (RV_OK, Some(asked), None)); + // SAFETY: as above. + let copy = in_step(|ctx| unsafe { file_copy(ctx, rv_str("/a"), rv_str("/b")) }); + let asked = VfsOp::Copy { + from: path("/a"), + to: path("/b"), + }; + assert_eq!(copy, (RV_OK, Some(asked), None)); +} + +#[test] +fn a_read_asks_for_no_offset_and_no_length_past_what_a_file_can_have() { + // SAFETY: the open step's `ctx` and a live path. + let read = in_step(|ctx| unsafe { file_read(ctx, rv_str("/a"), u64::MAX, u64::MAX) }); + let asked = VfsOp::ReadRange { + path: path("/a"), + offset: i64::MAX.unsigned_abs(), + len: u64::try_from(usize::MAX).unwrap_or(u64::MAX), + }; + assert_eq!(read, (RV_OK, Some(asked), None)); +} + +#[test] +fn each_breach_of_the_bytes_of_a_write_is_a_recorded_misuse() { + let byte = 0_u8; + for (bytes, misuse) in [ + ( + // The length is refused before any byte is read. + RvBytes { + ptr: &raw const byte, + len: 16_777_217, + }, + "the run of bytes is longer than 16777216 bytes", + ), + ( + RvBytes { + ptr: ptr::null(), + len: 2, + }, + "the run of bytes has a NULL pointer and a nonzero length", + ), + ] { + // SAFETY: the function refuses the bytes before it reads through + // them. + let write = in_step(|ctx| unsafe { file_write(ctx, rv_str("/a"), bytes) }); + assert_eq!(write, (RV_ERR_MISUSE, None, Some(misuse))); + // SAFETY: as above. + let append = in_step(|ctx| unsafe { file_append(ctx, rv_str("/a"), bytes) }); + assert_eq!(append, (RV_ERR_MISUSE, None, Some(misuse))); + } + let none = RvBytes { + ptr: ptr::null(), + len: 0, + }; + // SAFETY: a zero length reads no byte. + let empty = in_step(|ctx| unsafe { file_write(ctx, rv_str("/a"), none) }); + let asked = VfsOp::Write { + path: path("/a"), + contents: Vec::new(), + }; + assert_eq!(empty, (RV_OK, Some(asked), None)); +} + +#[test] +fn a_recursive_that_is_neither_0_nor_1_is_a_recorded_misuse() { + let misuse = Some("recursive is neither 0 nor 1"); + // SAFETY: the open step's `ctx` and a live path. + let mkdir = in_step(|ctx| unsafe { file_mkdir(ctx, rv_str("/a"), 2) }); + assert_eq!(mkdir, (RV_ERR_MISUSE, None, misuse)); + // SAFETY: as above. + let remove = in_step(|ctx| unsafe { file_remove(ctx, rv_str("/a"), u32::MAX) }); + assert_eq!(remove, (RV_ERR_MISUSE, None, misuse)); +} + +#[test] +fn each_breach_of_either_path_of_a_move_or_a_copy_is_a_recorded_misuse() { + let long = "a".repeat(4097); + let bad = rv_str_bytes(&[0xff, 0xfe]); + for (from, to, misuse) in [ + ( + rv_str(&long), + rv_str("/b"), + "the source path is longer than 4096 bytes", + ), + ( + rv_str("/a"), + rv_str(&long), + "the destination path is longer than 4096 bytes", + ), + (bad, rv_str("/b"), "the source path is not UTF-8"), + (rv_str("/a"), bad, "the destination path is not UTF-8"), + // The arguments are checked from left to right. + (bad, rv_str(&long), "the source path is not UTF-8"), + ] { + // SAFETY: the open step's `ctx` and two paths that are live for + // their lengths. + let rename = in_step(|ctx| unsafe { file_rename(ctx, from, to) }); + assert_eq!(rename, (RV_ERR_MISUSE, None, Some(misuse))); + // SAFETY: as above. + let copy = in_step(|ctx| unsafe { file_copy(ctx, from, to) }); + assert_eq!(copy, (RV_ERR_MISUSE, None, Some(misuse))); + } +} + +#[test] +fn each_breach_of_a_path_is_a_recorded_misuse_and_takes_no_request() { + let long = "a".repeat(4097); + for (path, misuse) in [ + (rv_str(&long), "the path is longer than 4096 bytes"), + ( + RvStr { + ptr: ptr::null(), + len: 2, + }, + "the path has a NULL pointer and a nonzero length", + ), + (rv_str_bytes(&[0xff, 0xfe]), "the path is not UTF-8"), + ] { + // SAFETY: the function refuses the path before it reads through + // it, or the path is live. + let stat = in_step(|ctx| unsafe { file_stat(ctx, path) }); + assert_eq!(stat, (RV_ERR_MISUSE, None, Some(misuse))); + // The path of a write is checked before its bytes. + let bytes = RvBytes { + ptr: ptr::null(), + len: 2, + }; + // SAFETY: as above, and the bytes are never reached. + let write = in_step(|ctx| unsafe { file_write(ctx, path, bytes) }); + assert_eq!(write, (RV_ERR_MISUSE, None, Some(misuse))); + } +} + +#[test] +fn a_second_request_in_one_step_is_a_recorded_misuse_and_the_first_stays() { + // SAFETY: the open step's `ctx` and live paths. + let (statuses, request, misuse) = in_step(|ctx| unsafe { + ( + file_stat(ctx, rv_str("/a")), + file_read(ctx, rv_str("/b"), 0, 1), + ) + }); + assert_eq!(statuses, (RV_OK, RV_ERR_MISUSE)); + assert_eq!(request, Some(VfsOp::Stat { path: path("/a") })); + assert_eq!(misuse, Some("a step made a second file request")); +} + +#[test] +fn a_request_and_finish_in_one_step_are_a_recorded_misuse_in_either_order() { + let done = result(RV_RESULT_TRUSTED, rv_str("done")); + // SAFETY: the open step's `ctx`, a live path, and a live result. + let (statuses, request, misuse) = in_step(|ctx| unsafe { + ( + crate::call::finish(ctx, &raw const done), + file_stat(ctx, rv_str("/a")), + ) + }); + assert_eq!(statuses, (RV_OK, RV_ERR_MISUSE)); + assert_eq!(request, None); + assert_eq!( + misuse, + Some("a step made a file request after it called finish") + ); + + // SAFETY: as above. + let (statuses, request, misuse) = in_step(|ctx| unsafe { + ( + file_stat(ctx, rv_str("/a")), + crate::call::finish(ctx, &raw const done), + ) + }); + assert_eq!(statuses, (RV_OK, RV_ERR_MISUSE)); + assert_eq!(request, Some(VfsOp::Stat { path: path("/a") })); + assert_eq!( + misuse, + Some("a step called finish after it made a file request") + ); +} + +#[test] +fn a_step_that_may_not_ask_is_refused_before_its_arguments_are_read() { + let done = result(RV_RESULT_TRUSTED, rv_str("done")); + let null = RvStr { + ptr: ptr::null(), + len: 2, + }; + // SAFETY: the open step's `ctx` and a live result, and the function + // refuses the step before it reads the path. + let (_statuses, request, misuse) = in_step(|ctx| unsafe { + ( + crate::call::finish(ctx, &raw const done), + file_stat(ctx, null), + ) + }); + assert_eq!(request, None); + assert_eq!( + misuse, + Some("a step made a file request after it called finish") + ); + + // SAFETY: the open step's `ctx` and a live path, and the second + // function refuses the step before it reads its path. + let (statuses, _request, misuse) = + in_step(|ctx| unsafe { (file_stat(ctx, rv_str("/a")), file_stat(ctx, null)) }); + assert_eq!(statuses, (RV_OK, RV_ERR_MISUSE)); + assert_eq!(misuse, Some("a step made a second file request")); +} + +#[test] +fn a_file_function_answers_only_the_open_step_through_its_own_ctx() { + let path = rv_str("/notes/a.md"); + + // SAFETY: a `ctx` that is not the open step's is refused before + // anything is read through it. + let wrong_ctx = in_step(|_ctx| unsafe { file_stat(ptr::null_mut(), path) }); + assert_eq!( + wrong_ctx, + ( + RV_ERR_MISUSE, + None, + Some("a tool call function was handed a ctx other than the open RV_TOOL_CALL step's") + ) + ); + + let (status, session) = session::open(Verb::PackNew, |ctx| { + // SAFETY: a call that is not a step is refused before anything + // is read. + unsafe { file_stat(ctx, path) } + }); + assert_eq!( + (status, session.misuse), + ( + RV_ERR_MISUSE, + Some("a tool call function was called outside RV_TOOL_CALL") + ) + ); + assert_eq!(session.request, None); + + // SAFETY: with no call open, the function reads nothing. + let closed = session::closed(|| unsafe { file_stat(ptr::null_mut(), path) }); + assert_eq!(closed, RV_ERR_MISUSE); +} + +#[test] +fn each_request_asks_for_the_one_field_its_file_function_names() { + let path = || "/a".to_owned(); + let contents = || b"x".to_vec(); + for (op, wanted) in [ + (VfsOp::Stat { path: path() }, Wanted::Stat), + (VfsOp::List { path: path() }, Wanted::Entries), + ( + VfsOp::ReadRange { + path: path(), + offset: 2, + len: 7, + }, + Wanted::Bytes(7), + ), + ( + VfsOp::Write { + path: path(), + contents: contents(), + }, + Wanted::Nothing, + ), + ( + VfsOp::Append { + path: path(), + contents: contents(), + }, + Wanted::Nothing, + ), + ( + VfsOp::Mkdir { + path: path(), + recursive: true, + }, + Wanted::Nothing, + ), + ( + VfsOp::Remove { + path: path(), + recursive: false, + }, + Wanted::Nothing, + ), + ( + VfsOp::Rename { + from: path(), + to: path(), + }, + Wanted::Nothing, + ), + ( + VfsOp::Copy { + from: path(), + to: path(), + }, + Wanted::Nothing, + ), + ] { + assert_eq!(Wanted::by(&op), wanted, "{op:?}"); + } +} + +/// Whether `answer` holds no stat, no entries, and no bytes. +fn holds_nothing(answer: &RvFileAnswer) -> bool { + (answer.stat.file_type, answer.stat.reserved, answer.stat.len) == (0, 0, 0) + && answer.entries.ptr.is_null() + && answer.entries.len == 0 + && answer.bytes.ptr.is_null() + && answer.bytes.len == 0 +} + +/// Whether `answer` holds no error. +fn holds_no_error(answer: &RvFileAnswer) -> bool { + (answer.error.kind, answer.error.reserved) == (0, 0) + && answer.error.text.ptr.is_null() + && answer.error.text.len == 0 +} + +#[test] +fn the_answer_to_a_stat_holds_the_stat_and_nothing_else() { + let found = VfsOutcome::Stat(Stat::new(FileType::File, 5)); + let answer = Answer::to(Wanted::Stat, Ok(found)); + let raw = answer.raw(); + assert_eq!((raw.size, raw.status), (size_u32::(), RV_OK)); + assert_eq!( + (raw.stat.file_type, raw.stat.reserved, raw.stat.len), + (RV_FILE_TYPE_FILE, 0, 5) + ); + assert!(raw.entries.ptr.is_null() && raw.entries.len == 0); + assert!(raw.bytes.ptr.is_null() && raw.bytes.len == 0); + assert!(holds_no_error(&raw)); +} + +#[test] +fn the_answer_to_a_listing_holds_each_entry_with_its_name() { + let listed = VfsOutcome::Entries(vec![ + Entry::new("a.md", Stat::new(FileType::File, 5)), + Entry::new("sub", Stat::new(FileType::Directory, 0)), + ]); + let answer = Answer::to(Wanted::Entries, Ok(listed)); + let raw = answer.raw(); + assert_eq!(raw.status, RV_OK); + // SAFETY: the entries and their names are valid while `answer` + // lives. + let entries: Vec<(String, u32, u64)> = + unsafe { slice::from_raw_parts(raw.entries.ptr, raw.entries.len) } + .iter() + .map(|entry| (text(entry.name), entry.stat.file_type, entry.stat.len)) + .collect(); + assert_eq!( + entries, + [ + ("a.md".to_owned(), RV_FILE_TYPE_FILE, 5), + ("sub".to_owned(), RV_FILE_TYPE_DIRECTORY, 0), + ] + ); + assert_eq!( + (raw.stat.file_type, raw.stat.reserved, raw.stat.len), + (0, 0, 0) + ); + assert!(raw.bytes.ptr.is_null() && raw.bytes.len == 0); + assert!(holds_no_error(&raw)); + + let empty = Answer::to(Wanted::Entries, Ok(VfsOutcome::Entries(Vec::new()))); + let raw = empty.raw(); + assert_eq!(raw.status, RV_OK); + assert!(holds_nothing(&raw) && holds_no_error(&raw)); +} + +#[test] +fn the_answer_to_a_read_holds_at_most_the_bytes_the_request_asked_for() { + let read = || Ok(VfsOutcome::Bytes(b"hello".to_vec())); + for (most, held) in [(u64::MAX, &b"hello"[..]), (5, b"hello"), (3, b"hel")] { + let answer = Answer::to(Wanted::Bytes(most), read()); + let raw = answer.raw(); + assert_eq!(raw.status, RV_OK); + // SAFETY: the bytes are valid while `answer` lives. + let bytes = unsafe { slice::from_raw_parts(raw.bytes.ptr, raw.bytes.len) }; + assert_eq!(bytes, held, "at most {most} bytes"); + assert!(raw.entries.ptr.is_null() && raw.entries.len == 0); + assert!(holds_no_error(&raw)); + } +} + +#[test] +fn the_answer_to_a_read_of_no_bytes_holds_nothing() { + for answer in [ + Answer::to(Wanted::Bytes(9), Ok(VfsOutcome::Bytes(Vec::new()))), + Answer::to(Wanted::Bytes(0), Ok(VfsOutcome::Bytes(b"hello".to_vec()))), + ] { + let raw = answer.raw(); + assert_eq!(raw.status, RV_OK); + assert!(holds_nothing(&raw) && holds_no_error(&raw)); + } +} + +#[test] +fn the_answer_to_a_failed_operation_holds_the_error_and_nothing_else() { + let absent = VfsError::NotFound { + path: "/absent".to_owned(), + }; + for wanted in [ + Wanted::Stat, + Wanted::Entries, + Wanted::Bytes(9), + Wanted::Nothing, + ] { + let answer = Answer::to(wanted, Err(absent.clone())); + let raw = answer.raw(); + assert_eq!(raw.status, RV_ERR_FILE, "{wanted:?}"); + assert_eq!( + (raw.error.kind, raw.error.reserved, text(raw.error.text)), + (RV_FILE_ERROR_NOT_FOUND, 0, "not found: /absent".to_owned()), + "{wanted:?}" + ); + assert!(holds_nothing(&raw), "{wanted:?}"); + } +} + +#[test] +fn the_answer_to_a_change_holds_a_status_alone() { + let raw = Answer::to(Wanted::Nothing, Ok(VfsOutcome::Unit)).raw(); + assert_eq!((raw.size, raw.status), (size_u32::(), RV_OK)); + assert!(holds_nothing(&raw) && holds_no_error(&raw)); +} + +#[test] +fn an_outcome_of_another_kind_than_the_request_asked_for_is_a_failed_operation() { + let stat = || VfsOutcome::Stat(Stat::new(FileType::File, 5)); + for (wanted, outcome) in [ + (Wanted::Stat, VfsOutcome::Unit), + (Wanted::Entries, stat()), + (Wanted::Bytes(9), VfsOutcome::Text("hello".to_owned())), + (Wanted::Nothing, stat()), + ] { + let answer = Answer::to(wanted, Ok(outcome)); + let raw = answer.raw(); + assert_eq!(raw.status, RV_ERR_FILE, "{wanted:?}"); + assert_eq!( + (raw.error.kind, text(raw.error.text)), + ( + RV_FILE_ERROR_OTHER, + "the file operation returned something other than what was asked for".to_owned() + ), + "{wanted:?}" + ); + assert!(holds_nothing(&raw), "{wanted:?}"); + } +} + +#[test] +fn each_kind_of_node_keeps_its_number_and_any_other_kind_is_other() { + for (file_type, raw) in [ + (FileType::File, RV_FILE_TYPE_FILE), + (FileType::Directory, RV_FILE_TYPE_DIRECTORY), + (FileType::Symlink, RV_FILE_TYPE_SYMLINK), + (FileType::Fifo, RV_FILE_TYPE_OTHER), + (FileType::Socket, RV_FILE_TYPE_OTHER), + (FileType::CharDevice, RV_FILE_TYPE_OTHER), + (FileType::BlockDevice, RV_FILE_TYPE_OTHER), + ] { + let stat = raw_stat(&Stat::new(file_type, 7)); + assert_eq!( + (stat.file_type, stat.reserved, stat.len), + (raw, 0, 7), + "{file_type:?}" + ); + } +} + +#[test] +fn each_error_the_contract_has_a_kind_for_reads_as_that_kind_and_a_text_a_model_can_read() { + let path = || "/x".to_owned(); + for (error, kind, text) in [ + ( + VfsError::NotFound { path: path() }, + RV_FILE_ERROR_NOT_FOUND, + "not found: /x", + ), + ( + VfsError::AlreadyExists { path: path() }, + RV_FILE_ERROR_ALREADY_EXISTS, + "already exists: /x", + ), + ( + VfsError::NotADirectory { path: path() }, + RV_FILE_ERROR_NOT_A_DIRECTORY, + "not a directory: /x", + ), + ( + VfsError::IsADirectory { path: path() }, + RV_FILE_ERROR_IS_A_DIRECTORY, + "is a directory: /x", + ), + ( + VfsError::DirectoryNotEmpty { path: path() }, + RV_FILE_ERROR_NOT_EMPTY, + "directory not empty: /x", + ), + ( + VfsError::PermissionDenied { + path: path(), + reason: "the mode allows only markdown".to_owned(), + }, + RV_FILE_ERROR_DENIED, + "the mode allows only markdown", + ), + ( + VfsError::ApprovalRequired { + path: path(), + reason: "writing /x needs approval".to_owned(), + }, + RV_FILE_ERROR_DENIED, + "writing /x needs approval", + ), + ( + VfsError::Io { + path: path(), + kind: io::ErrorKind::PermissionDenied, + raw_os_error: Some(13), + message: "access denied".to_owned(), + }, + RV_FILE_ERROR_IO, + "permission denied: access denied", + ), + ] { + assert_eq!(describe(&error), (kind, text.to_owned()), "{error:?}"); + } +} + +#[test] +fn each_error_the_contract_has_no_kind_for_reads_as_other_and_a_text_a_model_can_read() { + let path = || "/x".to_owned(); + for (error, text) in [ + ( + // The files a tool call reaches check no conflict, so no + // error reads as a conflict, one a backend makes included. + VfsError::Conflict { + path: path(), + detail: "a write by another call".to_owned(), + }, + "a write by another call", + ), + ( + VfsError::InvalidPath { + path: String::new(), + reason: PathReason::Empty, + }, + "path is empty", + ), + ( + // The path holds the `": "` that ends a label. + VfsError::InvalidPath { + path: "a: b/../..".to_owned(), + reason: PathReason::Traversal, + }, + "path contains a traversal segment", + ), + ( + VfsError::NotUtf8 { + path: path(), + valid_up_to: 3, + }, + "/x holds a name or contents that are not UTF-8", + ), + ( + VfsError::ScopeEnded { path: path() }, + "the run or tool call that owned this access has ended, so /x can no longer be \ + touched through it", + ), + ( + VfsError::ReadOnly { path: path() }, + "/x is served read-only and cannot change", + ), + ( + VfsError::Backend { + message: "the backend is gone".to_owned(), + }, + "the backend is gone", + ), + ] { + assert_eq!( + describe(&error), + (RV_FILE_ERROR_OTHER, text.to_owned()), + "{error:?}" + ); + } +} + pack_fake!( reader, {}, @@ -34,12 +711,23 @@ pack_fake!( let args = step_args(call); // The arguments are one JSON string, the path. let path: String = serde_json::from_str(&text(args.arguments)).unwrap_or_default(); - let said = match text(args.tool).as_str() { - "stat" => format!("{:?}", stat_of(call, &path)), - "list" => format!("{:?}", list_of(call, &path)), - "read" => format!("{:?}", read_of(call, &path, (0, u64::MAX))), - "tail" => format!("{:?}", read_of(call, &path, (3, 100))), - _ => format!("{:?}", read_of(call, &path, (u64::MAX, u64::MAX))), + let tool = text(args.tool); + if tool_call(call).is_null() { + // The first step asks, and the second is lent the answer. + set_tool_call(call, instance(0x300)); + let asked = match tool.as_str() { + "stat" => ask_stat(call, &path), + "list" => ask_list(call, &path), + "read" => ask_read(call, &path, (0, u64::MAX)), + "tail" => ask_read(call, &path, (3, 100)), + _ => ask_read(call, &path, (u64::MAX, u64::MAX)), + }; + return if asked == RV_OK { RV_PENDING } else { ODD_STATUS }; + } + let said = match tool.as_str() { + "stat" => format!("{:?}", stat_answer(call)), + "list" => format!("{:?}", list_answer(call)), + _ => format!("{:?}", read_answer(call)), }; finish(call, RV_RESULT_TRUSTED, &said) } @@ -55,52 +743,62 @@ fn said(tool: &str, path: &str) -> String { .to_owned() } -/// The text a fake's helper formats for a failure of `kind` that says -/// `text`. +/// The text a fake's helper formats for an answer that holds `held`. +fn held(held: &str) -> String { + format!("Some(Ok({held}))") +} + +/// The text a fake's helper formats for the answer of an operation that +/// failed as `kind`, saying `text`. fn failed(kind: u32, text: &str) -> String { - format!("Err(({kind}, {text:?}))") + format!("Some(Err(({kind}, {text:?})))") } #[test] -fn file_stat_says_what_a_path_names() { +fn a_stat_is_answered_with_what_the_path_names() { assert_eq!( said("stat", "/notes/a.md"), - format!("Ok(({RV_FILE_TYPE_FILE}, 5))") + held(&format!("({RV_FILE_TYPE_FILE}, 5)")) ); assert!( - said("stat", "/notes").starts_with(&format!("Ok(({RV_FILE_TYPE_DIRECTORY}, ")), + said("stat", "/notes").starts_with(&format!("Some(Ok(({RV_FILE_TYPE_DIRECTORY}, ")), "{}", said("stat", "/notes") ); } #[test] -fn file_list_hands_back_each_entry_with_its_name_and_metadata() { +fn a_listing_is_answered_with_each_entry_its_name_and_its_metadata() { assert_eq!( said("list", "/notes"), - format!("Ok([(\"a.md\", {RV_FILE_TYPE_FILE}, 5), (\"b.md\", {RV_FILE_TYPE_FILE}, 0)])") + held(&format!( + "[(\"a.md\", {RV_FILE_TYPE_FILE}, 5), (\"b.md\", {RV_FILE_TYPE_FILE}, 0)]" + )) ); } #[test] -fn file_read_hands_back_the_whole_file_or_the_bytes_inside_a_range() { - assert_eq!(said("read", "/notes/a.md"), format!("Ok({:?})", b"hello")); - assert_eq!(said("read", "/notes/b.md"), "Ok([])"); - assert_eq!(said("tail", "/notes/a.md"), format!("Ok({:?})", b"lo")); +fn a_read_is_answered_with_the_whole_file_or_the_bytes_inside_a_range() { + assert_eq!( + said("read", "/notes/a.md"), + held(&format!("{:?}", b"hello")) + ); + assert_eq!(said("read", "/notes/b.md"), held("[]")); + assert_eq!(said("tail", "/notes/a.md"), held(&format!("{:?}", b"lo"))); assert_eq!( said("past", "/notes/a.md"), - "Ok([])", + held("[]"), "the largest offset is past the end, not a failure" ); } #[test] fn a_path_without_a_leading_slash_is_taken_from_the_root() { - assert_eq!(said("read", "notes/a.md"), format!("Ok({:?})", b"hello")); + assert_eq!(said("read", "notes/a.md"), held(&format!("{:?}", b"hello"))); } #[test] -fn a_failed_file_function_fills_the_error_and_leaves_the_call_running() { +fn a_failed_operation_is_answered_with_its_error_and_leaves_the_call_running() { assert_eq!( said("stat", "/absent"), failed(RV_FILE_ERROR_NOT_FOUND, "not found: /absent") @@ -132,181 +830,203 @@ fn a_path_of_exactly_4096_bytes_is_within_the_bound() { } pack_fake!( - twice, + changer, {}, new: |call| { set_pack_run(call, instance(PACK_TAG)); - add_tool(call, "twice") + add_tool(call, "change") }, step: |call| { - let read = read_of(call, "/notes/a.md", (0, u64::MAX)); + // The arguments are a JSON array of three strings: the change, + // a path, and a text to write or a second path. + let (change, first, second): (String, String, String) = + serde_json::from_str(&text(step_args(call).arguments)).unwrap_or_default(); if tool_call(call).is_null() { + // The first step asks, and the second is lent the answer. set_tool_call(call, instance(0x300)); - return if read.is_ok() { RV_PENDING } else { ODD_STATUS }; + let asked = match change.as_str() { + "write" => ask_write(call, &first, rv_bytes(second.as_bytes())), + "append" => ask_append(call, &first, rv_bytes(second.as_bytes())), + "mkdir" => ask_mkdir(call, &first, 0), + "mkdirs" => ask_mkdir(call, &first, 1), + "remove" => ask_remove(call, &first, 0), + "purge" => ask_remove(call, &first, 1), + "rename" => ask_rename(call, rv_str(&first), rv_str(&second)), + _ => ask_copy(call, rv_str(&first), rv_str(&second)), + }; + return if asked == RV_OK { RV_PENDING } else { ODD_STATUS }; } - finish(call, RV_RESULT_TRUSTED, &format!("{read:?}")) + finish(call, RV_RESULT_TRUSTED, &format!("{:?}", change_answer(call))) } ); -#[test] -fn each_step_of_a_call_reads_through_its_own_ctx() { - let pack = made(twice::entry); - let call = seeded_call("twice", FILES); - let output = block_on(pack.call(call.context(), json!({}))).expect("the call succeeds"); - assert_eq!(output.text(), format!("Ok({:?})", b"hello")); +/// What the `changer` Pack says of the change `change`, with `first` and +/// `second` as its arguments, on the files of `call`. +fn change_said(call: &TestCall, change: &str, first: &str, second: &str) -> String { + let pack = made(changer::entry); + block_on(pack.call(call.context(), json!([change, first, second]))) + .expect("the call succeeds") + .text() + .to_owned() } #[test] -fn each_kind_of_node_keeps_its_number_and_any_other_kind_is_other() { - for (file_type, raw) in [ - (FileType::File, RV_FILE_TYPE_FILE), - (FileType::Directory, RV_FILE_TYPE_DIRECTORY), - (FileType::Symlink, RV_FILE_TYPE_SYMLINK), - (FileType::Fifo, RV_FILE_TYPE_OTHER), - (FileType::Socket, RV_FILE_TYPE_OTHER), - (FileType::CharDevice, RV_FILE_TYPE_OTHER), - (FileType::BlockDevice, RV_FILE_TYPE_OTHER), - ] { - let stat = raw_stat(&Stat::new(file_type, 7)); - assert_eq!( - (stat.file_type, stat.reserved, stat.len), - (raw, 0, 7), - "{file_type:?}" - ); - } -} +fn each_change_is_performed_on_the_files_of_the_call_and_answered_with_a_status_alone() { + let call = seeded_call("change", FILES); + let done = held("()"); -#[test] -fn a_panic_inside_a_file_operation_is_a_failure_and_never_an_unwind() { - let caught: Result<(), VfsError> = contained(|| panic!("the policy broke")); - let failure = caught.expect_err("the panic is a failure"); + assert_eq!(change_said(&call, "write", "/out/new.md", "one"), done); + assert_eq!(text_at(&call, "/out/new.md"), Ok("one".to_owned())); + assert_eq!(change_said(&call, "append", "/out/new.md", " two"), done); + assert_eq!(text_at(&call, "/out/new.md"), Ok("one two".to_owned())); + assert_eq!(change_said(&call, "append", "/out/added.md", "made"), done); + assert_eq!(text_at(&call, "/out/added.md"), Ok("made".to_owned())); assert_eq!( - describe(&failure), - ( - RV_FILE_ERROR_OTHER, - "the file operation panicked".to_owned() - ) + change_said(&call, "copy", "/out/new.md", "/deep/er/copy.md"), + done + ); + assert_eq!(text_at(&call, "/deep/er/copy.md"), Ok("one two".to_owned())); + assert_eq!( + change_said(&call, "rename", "/out/new.md", "/moved/new.md"), + done + ); + assert_eq!(text_at(&call, "/moved/new.md"), Ok("one two".to_owned())); + assert_eq!( + text_at(&call, "/out/new.md"), + Err(VfsError::NotFound { + path: "/out/new.md".to_owned() + }) + ); + assert_eq!(change_said(&call, "mkdirs", "/made/in/depth", ""), done); + assert_eq!(change_said(&call, "mkdir", "/made/in/depth/more", ""), done); + assert_eq!( + change_said(&call, "remove", "/made/in/depth/more", ""), + done + ); + assert_eq!(change_said(&call, "purge", "/notes", ""), done); + assert_eq!( + text_at(&call, "/notes/a.md"), + Err(VfsError::NotFound { + path: "/notes/a.md".to_owned() + }) ); - assert_eq!(contained(|| Ok(7)), Ok(7), "any other outcome passes"); } #[test] -fn each_error_reads_as_its_kind_and_a_text_a_model_can_read() { - let path = || "/x".to_owned(); - for (error, kind, text) in [ - ( - VfsError::NotFound { path: path() }, - RV_FILE_ERROR_NOT_FOUND, - "not found: /x", - ), - ( - VfsError::NotADirectory { path: path() }, - RV_FILE_ERROR_NOT_A_DIRECTORY, - "not a directory: /x", - ), +fn a_change_the_files_refuse_is_answered_with_its_kind_and_leaves_the_call_running() { + let call = seeded_call("change", FILES); + for (change, first, kind, text) in [ + // Something is at the path already, with `recursive` set too. ( - VfsError::IsADirectory { path: path() }, - RV_FILE_ERROR_IS_A_DIRECTORY, - "is a directory: /x", + "mkdir", + "/notes", + RV_FILE_ERROR_ALREADY_EXISTS, + "already exists: /notes", ), ( - VfsError::PermissionDenied { - path: path(), - reason: "the mode allows only markdown".to_owned(), - }, - RV_FILE_ERROR_DENIED, - "the mode allows only markdown", + "mkdirs", + "/notes", + RV_FILE_ERROR_ALREADY_EXISTS, + "already exists: /notes", ), ( - VfsError::ApprovalRequired { - path: path(), - reason: "reading /x needs approval".to_owned(), - }, - RV_FILE_ERROR_DENIED, - "reading /x needs approval", + "mkdirs", + "/notes/a.md", + RV_FILE_ERROR_ALREADY_EXISTS, + "already exists: /notes/a.md", ), + // A directory on the way is absent, and `recursive` is 0. ( - VfsError::Conflict { - path: path(), - detail: "a write by another call".to_owned(), - }, - RV_FILE_ERROR_CONFLICT, - "another tool call of the run changed /x, or is changing it", + "mkdir", + "/no/such", + RV_FILE_ERROR_NOT_FOUND, + "not found: /no/such", ), + // Nothing is at the path. ( - VfsError::Io { - path: path(), - kind: io::ErrorKind::PermissionDenied, - raw_os_error: Some(13), - message: "access denied".to_owned(), - }, - RV_FILE_ERROR_IO, - "permission denied: access denied", + "remove", + "/absent", + RV_FILE_ERROR_NOT_FOUND, + "not found: /absent", ), ( - VfsError::InvalidPath { - path: String::new(), - reason: PathReason::Empty, - }, - RV_FILE_ERROR_OTHER, - "path is empty", + "purge", + "/absent", + RV_FILE_ERROR_NOT_FOUND, + "not found: /absent", ), + // The directory has entries, and `recursive` is 0. ( - // The path holds the `": "` that ends a label. - VfsError::InvalidPath { - path: "a: b/../..".to_owned(), - reason: PathReason::Traversal, - }, - RV_FILE_ERROR_OTHER, - "path contains a traversal segment", + "remove", + "/notes", + RV_FILE_ERROR_NOT_EMPTY, + "directory not empty: /notes", ), ( - VfsError::NotUtf8 { - path: path(), - valid_up_to: 3, - }, - RV_FILE_ERROR_OTHER, - "/x holds a name or contents that are not UTF-8", + "write", + "/notes", + RV_FILE_ERROR_IS_A_DIRECTORY, + "is a directory: /notes", ), ( - VfsError::ScopeEnded { path: path() }, - RV_FILE_ERROR_OTHER, - "the run or tool call that owned this access has ended, so /x can no longer be \ - touched through it", + "rename", + "/absent", + RV_FILE_ERROR_NOT_FOUND, + "not found: /absent", ), ( - VfsError::Backend { - message: "the backend is gone".to_owned(), - }, - RV_FILE_ERROR_OTHER, - "the backend is gone", + "copy", + "/absent", + RV_FILE_ERROR_NOT_FOUND, + "not found: /absent", ), ] { - assert_eq!(describe(&error), (kind, text.to_owned()), "{error:?}"); + assert_eq!( + change_said(&call, change, first, "/other"), + failed(kind, text), + "{change} {first}" + ); } -} - -#[test] -fn each_file_function_keeps_only_what_it_hands_back() { - let mut lent = Lent::default(); - let bytes = lent.bytes(b"hello".to_vec()); - assert_eq!((bytes.len, lent.bytes.len()), (5, 5)); - - let failure = lent.error(&VfsError::NotFound { - path: "/x".to_owned(), - }); assert_eq!( - (failure.kind, failure.reserved, text(failure.text)), - (RV_FILE_ERROR_NOT_FOUND, 0, "not found: /x".to_owned()) + text_at(&call, "/notes/a.md"), + Ok("hello".to_owned()), + "a refused change left the files as they were" ); - assert!(lent.bytes.is_empty(), "the bytes of the read are gone"); +} - let listed = lent.entries(vec![runvil_plugin::Entry::new( - "a.md", - Stat::new(FileType::File, 5), - )]); - assert_eq!((listed.len, lent.entries.len()), (1, 1)); - assert!(lent.text.is_empty(), "the text of the failure is gone"); +pack_fake!( + twice, + {}, + new: |call| { + set_pack_run(call, instance(PACK_TAG)); + add_tool(call, "twice") + }, + step: |call| { + // The call's state counts its steps. + let steps = tool_call(call).addr(); + set_tool_call(call, ptr::without_provenance_mut(steps + 1)); + match steps { + 0 => { + ask_read(call, "/notes/a.md", (0, u64::MAX)); + RV_PENDING + } + 1 => { + // The step that is lent an answer may ask again. + let first = read_answer(call); + ask_stat(call, "/notes/b.md"); + if first == Some(Ok(b"hello".to_vec())) { RV_PENDING } else { ODD_STATUS } + } + _ => finish(call, RV_RESULT_TRUSTED, &format!("{:?}", stat_answer(call))), + } + } +); + +#[test] +fn a_call_asks_in_one_step_after_another_and_each_next_step_is_lent_the_answer() { + let pack = made(twice::entry); + let call = seeded_call("twice", FILES); + let output = block_on(pack.call(call.context(), json!({}))).expect("the call succeeds"); + assert_eq!(output.text(), held(&format!("({RV_FILE_TYPE_FILE}, 0)"))); } /// The message of a call that failed because its Plugin misused it. @@ -343,290 +1063,74 @@ macro_rules! file_misuse_test { } file_misuse_test!( - a_path_over_4096_bytes_is_misuse, + a_path_over_4096_bytes_fails_the_call, "the path is longer than 4096 bytes", |call| { - let (mut stat, mut error) = (unset_stat(), unset_error()); // The length is refused before any byte is read, so the text // behind it need not be that long. let long = RvStr { ptr: "x".as_ptr().cast(), len: 4097, }; - stat_as( - call, - step_args(call).ctx, - long, - &raw mut stat, - &raw mut error, - ) - } -); - -file_misuse_test!( - a_path_with_a_null_pointer_and_a_length_is_misuse, - "the path has a NULL pointer and a nonzero length", - |call| { - let (mut stat, mut error) = (unset_stat(), unset_error()); - let null = RvStr { - ptr: ptr::null(), - len: 4, - }; - stat_as( - call, - step_args(call).ctx, - null, - &raw mut stat, - &raw mut error, - ) - } -); - -file_misuse_test!( - a_path_that_is_not_utf8_is_misuse, - "the path is not UTF-8", - |call| { - let (mut stat, mut error) = (unset_stat(), unset_error()); - let path = rv_str_bytes(&[0xff, 0xfe]); - stat_as( - call, - step_args(call).ctx, - path, - &raw mut stat, - &raw mut error, - ) + ask_stat_as(call, step_args(call).ctx, long) } ); file_misuse_test!( - a_file_function_through_another_ctx_is_misuse, + a_file_function_through_another_ctx_fails_the_call, "a tool call function was handed a ctx other than the open RV_TOOL_CALL step's", - |call| { - let (mut stat, mut error) = (unset_stat(), unset_error()); - stat_as( - call, - ptr::null_mut(), - rv_str("/notes/a.md"), - &raw mut stat, - &raw mut error, - ) - } -); - -file_misuse_test!( - a_null_stat_is_misuse, - "file_stat was handed a NULL stat pointer", - |call| { - let mut error = unset_error(); - stat_as( - call, - step_args(call).ctx, - rv_str("/notes/a.md"), - ptr::null_mut(), - &raw mut error, - ) - } -); - -file_misuse_test!( - a_misaligned_stat_is_misuse, - "file_stat was handed a misaligned stat pointer", - |call| { - let mut error = unset_error(); - let odd = ptr::without_provenance_mut(1); - stat_as( - call, - step_args(call).ctx, - rv_str("/notes/a.md"), - odd, - &raw mut error, - ) - } + |call| ask_stat_as(call, ptr::null_mut(), rv_str("/notes/a.md")) ); file_misuse_test!( - a_null_error_of_a_stat_is_misuse, - "file_stat was handed a NULL error pointer", + a_write_of_more_than_16_mebibytes_fails_the_call, + "the run of bytes is longer than 16777216 bytes", |call| { - let mut stat = unset_stat(); - stat_as( - call, - step_args(call).ctx, - rv_str("/notes/a.md"), - &raw mut stat, - ptr::null_mut(), - ) - } -); - -file_misuse_test!( - a_misaligned_error_of_a_stat_is_misuse, - "file_stat was handed a misaligned error pointer", - |call| { - let mut stat = unset_stat(); - let odd = ptr::without_provenance_mut(1); - stat_as( - call, - step_args(call).ctx, - rv_str("/notes/a.md"), - &raw mut stat, - odd, - ) - } -); - -file_misuse_test!( - a_null_entries_is_misuse, - "file_list was handed a NULL entries pointer", - |call| { - let mut error = unset_error(); - list_as( - call, - step_args(call).ctx, - rv_str("/notes"), - ptr::null_mut(), - &raw mut error, - ) - } -); - -file_misuse_test!( - a_misaligned_entries_is_misuse, - "file_list was handed a misaligned entries pointer", - |call| { - let mut error = unset_error(); - let odd = ptr::without_provenance_mut(1); - list_as( - call, - step_args(call).ctx, - rv_str("/notes"), - odd, - &raw mut error, - ) - } -); - -file_misuse_test!( - a_null_error_of_a_list_is_misuse, - "file_list was handed a NULL error pointer", - |call| { - let mut entries = unset_entries(); - list_as( - call, - step_args(call).ctx, - rv_str("/notes"), - &raw mut entries, - ptr::null_mut(), - ) - } -); - -file_misuse_test!( - a_misaligned_error_of_a_list_is_misuse, - "file_list was handed a misaligned error pointer", - |call| { - let mut entries = unset_entries(); - let odd = ptr::without_provenance_mut(1); - list_as( - call, - step_args(call).ctx, - rv_str("/notes"), - &raw mut entries, - odd, - ) - } -); - -file_misuse_test!( - a_null_bytes_is_misuse, - "file_read was handed a NULL bytes pointer", - |call| { - let mut error = unset_error(); - read_as( - call, - step_args(call).ctx, - rv_str("/notes/a.md"), - (0, 1), - ptr::null_mut(), - &raw mut error, - ) - } -); - -file_misuse_test!( - a_misaligned_bytes_is_misuse, - "file_read was handed a misaligned bytes pointer", - |call| { - let mut error = unset_error(); - let odd = ptr::without_provenance_mut(1); - read_as( - call, - step_args(call).ctx, - rv_str("/notes/a.md"), - (0, 1), - odd, - &raw mut error, - ) - } -); - -file_misuse_test!( - a_null_error_of_a_read_is_misuse, - "file_read was handed a NULL error pointer", - |call| { - let mut bytes = unset_bytes(); - read_as( - call, - step_args(call).ctx, - rv_str("/notes/a.md"), - (0, 1), - &raw mut bytes, - ptr::null_mut(), - ) + // The length is refused before any byte is read, so the bytes + // behind it need not be that many. + let long = RvBytes { + ptr: "x".as_ptr(), + len: 16_777_217, + }; + ask_write(call, "/notes/a.md", long) } ); file_misuse_test!( - a_misaligned_error_of_a_read_is_misuse, - "file_read was handed a misaligned error pointer", - |call| { - let mut bytes = unset_bytes(); - let odd = ptr::without_provenance_mut(1); - read_as( - call, - step_args(call).ctx, - rv_str("/notes/a.md"), - (0, 1), - &raw mut bytes, - odd, - ) - } + a_recursive_of_2_fails_the_call, + "recursive is neither 0 nor 1", + |call| ask_remove(call, "/notes", 2) ); #[test] -fn a_misuse_writes_neither_the_result_nor_the_error() { +fn a_misuse_asks_the_files_of_the_call_for_nothing() { pack_fake!( plug, - { - /// The file type and the error kind after a refused call. - pub(super) static LEFT: AtomicUsize = AtomicUsize::new(0); - }, + {}, new: |call| { set_pack_run(call, instance(PACK_TAG)); add_tool(call, "tool") }, step: |call| { - let (mut stat, mut error) = (unset_stat(), unset_error()); - let status = - stat_as(call, ptr::null_mut(), rv_str("/notes/a.md"), &raw mut stat, &raw mut error); - LEFT.store((stat.file_type + error.kind) as usize, Ordering::SeqCst); - status + // The write is a request the loader takes, and the second + // request of the step is the misuse. + ask_write(call, "/notes/a.md", rv_bytes(b"other")); + ask_stat(call, "/notes/a.md"); + RV_PENDING } ); let pack = made(plug::entry); let call = seeded_call("tool", FILES); - block_on(pack.call(call.context(), json!({}))).expect_err("the call fails"); - assert_eq!(plug::LEFT.load(Ordering::SeqCst), 18, "both still hold 9"); + let failure = block_on(pack.call(call.context(), json!({}))).expect_err("the call fails"); + assert_eq!( + failure.message, + misuse_reason!("a step made a second file request") + ); + assert_eq!( + text_at(&call, "/notes/a.md"), + Ok("hello".to_owned()), + "the request of a step that failed its call is not performed" + ); } #[test] @@ -635,17 +1139,9 @@ fn a_file_function_called_while_a_pack_instance_is_made_is_misuse() { plug, {}, new: |call| { - let (mut stat, mut error) = (unset_stat(), unset_error()); // SAFETY: the function refuses a call that is not a step - // before it reads or writes through any pointer. - unsafe { - crate::file::file_stat( - pack_args(call).ctx, - rv_str("/notes/a.md"), - &raw mut stat, - &raw mut error, - ) - } + // before it reads the path. + unsafe { crate::file::file_stat(pack_args(call).ctx, rv_str("/notes/a.md")) } }, step: |_call| RV_OK ); @@ -660,7 +1156,7 @@ fn a_file_function_called_while_a_pack_instance_is_made_is_misuse() { } #[test] -fn a_late_file_function_through_a_stashed_ctx_is_misuse_and_writes_nothing() { +fn a_late_file_function_through_a_stashed_ctx_is_misuse() { pack_fake!( plug, { @@ -679,17 +1175,8 @@ fn a_late_file_function_through_a_stashed_ctx_is_misuse_and_writes_nothing() { let call = seeded_call("tool", FILES); block_on(pack.call(call.context(), json!({}))).expect("the call succeeds"); let stashed = plug::STEP_CTX.load(Ordering::SeqCst); - let (mut stat, mut error) = (unset_stat(), unset_error()); // SAFETY: the loader's function compares a `ctx` with the open call's // before it reads through it, and no call is open on this thread. - let status = unsafe { - crate::file::file_stat( - stashed, - rv_str("/notes/a.md"), - &raw mut stat, - &raw mut error, - ) - }; + let status = unsafe { file_stat(stashed, rv_str("/notes/a.md")) }; assert_eq!(status, RV_ERR_MISUSE); - assert_eq!((stat.file_type, error.kind), (9, 9)); } diff --git a/crates/harness-internal/plugin-loader/src/file.rs b/crates/harness-internal/plugin-loader/src/file.rs index 6f92ad959..c88d62af9 100644 --- a/crates/harness-internal/plugin-loader/src/file.rs +++ b/crates/harness-internal/plugin-loader/src/file.rs @@ -1,39 +1,245 @@ -//! The file functions an `RvToolCallArgs` carries: `file_stat`, -//! `file_list`, and `file_read`, each answered from the access of the -//! step's tool call, so the run's policy applies to every read, and -//! [`Lent`], what the last of them handed back. +//! The nine file functions an `RvToolCallArgs` carries, each of which +//! takes one request of its step as a `VfsOp` and fills nothing, and +//! [`Answer`], what the loader lends the step after a request. use std::ffi::c_void; -use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::ptr; -use runvil_plugin::{Entry, FileType, Stat, VfsError}; +use runvil_plugin::{Entry, FileType, Stat, VfsError, VfsOp, VfsOutcome}; use runvil_plugin_sys::{ - RV_ERR_FILE, RV_FILE_ERROR_CONFLICT, RV_FILE_ERROR_DENIED, RV_FILE_ERROR_IO, - RV_FILE_ERROR_IS_A_DIRECTORY, RV_FILE_ERROR_NOT_A_DIRECTORY, RV_FILE_ERROR_NOT_FOUND, - RV_FILE_ERROR_OTHER, RV_FILE_TYPE_DIRECTORY, RV_FILE_TYPE_FILE, RV_FILE_TYPE_OTHER, - RV_FILE_TYPE_SYMLINK, RV_MAX_PATH, RV_OK, RvBytes, RvDirEntries, RvDirEntry, RvFileError, - RvFileErrorKind, RvStat, RvStatus, RvStr, + RV_ERR_FILE, RV_FILE_ERROR_ALREADY_EXISTS, RV_FILE_ERROR_DENIED, RV_FILE_ERROR_IO, + RV_FILE_ERROR_IS_A_DIRECTORY, RV_FILE_ERROR_NOT_A_DIRECTORY, RV_FILE_ERROR_NOT_EMPTY, + RV_FILE_ERROR_NOT_FOUND, RV_FILE_ERROR_OTHER, RV_FILE_TYPE_DIRECTORY, RV_FILE_TYPE_FILE, + RV_FILE_TYPE_OTHER, RV_FILE_TYPE_SYMLINK, RV_MAX_PATH, RV_MAX_WRITE, RV_OK, RvBytes, + RvDirEntries, RvDirEntry, RvFileAnswer, RvFileError, RvFileErrorKind, RvStat, RvStatus, RvStr, }; -use crate::session::{Field, Out, answer_file, field, out, rv_str}; +use crate::session::{Field, TOOL_CALL, answer, field, rv_str, size_field}; #[cfg(test)] #[path = "file-tests.rs"] mod tests; const PATH: Field = field!("the path", 4096, RV_MAX_PATH); -const STAT: Out = out!("file_stat", "stat"); -const STAT_ERROR: Out = out!("file_stat", "error"); -const ENTRIES: Out = out!("file_list", "entries"); -const LIST_ERROR: Out = out!("file_list", "error"); -const BYTES: Out = out!("file_read", "bytes"); -const READ_ERROR: Out = out!("file_read", "error"); +const FROM: Field = field!("the source path", 4096, RV_MAX_PATH); +const TO: Field = field!("the destination path", 4096, RV_MAX_PATH); +const BYTES: Field = field!("the run of bytes", 16_777_216, RV_MAX_WRITE); -/// What a step's last file function handed the Plugin. The contract -/// keeps it valid until the step's next file function or the end of the -/// step, so each file function replaces it, and the step's end frees it. -#[derive(Default)] -pub(crate) struct Lent { +/// What a `recursive` that is neither 0 nor 1 is called. +const RECURSIVE: &str = "recursive is neither 0 nor 1"; + +/// What a second file request in one step is called. +const SECOND_REQUEST: &str = "a step made a second file request"; +/// What a file request in a step that called `finish` is called. +const REQUEST_AFTER_FINISH: &str = "a step made a file request after it called finish"; +/// What `finish` in a step that made a file request is called. +pub(crate) const FINISH_AFTER_REQUEST: &str = "a step called finish after it made a file request"; + +/// Takes the operation `read` reads out of a file function's arguments, +/// as the one request of the open step; returns RV_OK, or RV_ERR_MISUSE +/// for a request the step may not make and for arguments `read` refuses. +/// The step's own state is checked first, so `read` runs only for a +/// request the step may make. +fn take(ctx: *mut c_void, read: impl FnOnce() -> Result) -> RvStatus { + answer(ctx, &TOOL_CALL, |session| { + if session.result.is_some() { + return Err(REQUEST_AFTER_FINISH); + } + if session.request.is_some() { + return Err(SECOND_REQUEST); + } + session.request = Some(read()?); + Ok(RV_OK) + }) +} + +/// Takes a request for what `path` names; returns RV_OK or +/// RV_ERR_MISUSE, and fills nothing. +pub(crate) unsafe extern "C" fn file_stat(ctx: *mut c_void, path: RvStr) -> RvStatus { + take(ctx, || { + // SAFETY: the contract makes `path` valid for its length during + // this call. + let path = unsafe { PATH.read(path) }?; + Ok(VfsOp::Stat { path }) + }) +} + +/// Takes a request for the entries of the directory at `path`; returns +/// RV_OK or RV_ERR_MISUSE, and fills nothing. +pub(crate) unsafe extern "C" fn file_list(ctx: *mut c_void, path: RvStr) -> RvStatus { + take(ctx, || { + // SAFETY: the contract makes `path` valid for its length during + // this call. + let path = unsafe { PATH.read(path) }?; + Ok(VfsOp::List { path }) + }) +} + +/// Takes a request for up to `len` bytes of the file at `path` from the +/// byte `offset` on; returns RV_OK or RV_ERR_MISUSE, and fills nothing. +pub(crate) unsafe extern "C" fn file_read( + ctx: *mut c_void, + path: RvStr, + offset: u64, + len: u64, +) -> RvStatus { + take(ctx, || { + // SAFETY: the contract makes `path` valid for its length during + // this call. + let path = unsafe { PATH.read(path) }?; + // A real file cannot be positioned past this offset, and no file + // is that long, so the read finds the end and returns no bytes. + let offset = offset.min(i64::MAX.unsigned_abs()); + // No read returns more than memory can hold, and a backend that + // slices a whole file refuses a length it cannot address. + let len = usize::try_from(len).unwrap_or(usize::MAX); + let len = u64::try_from(len).unwrap_or(u64::MAX); + Ok(VfsOp::ReadRange { path, offset, len }) + }) +} + +/// Takes a request to write `bytes` as the whole contents of the file at +/// `path`; returns RV_OK or RV_ERR_MISUSE. +pub(crate) unsafe extern "C" fn file_write( + ctx: *mut c_void, + path: RvStr, + bytes: RvBytes, +) -> RvStatus { + take(ctx, || { + // SAFETY: the contract makes `path` and `bytes` valid for their + // lengths during this call. + let (path, contents) = unsafe { (PATH.read(path)?, BYTES.bytes(bytes.ptr, bytes.len)?) }; + Ok(VfsOp::Write { path, contents }) + }) +} + +/// Takes a request to add `bytes` to the end of the file at `path`; +/// returns RV_OK or RV_ERR_MISUSE. +pub(crate) unsafe extern "C" fn file_append( + ctx: *mut c_void, + path: RvStr, + bytes: RvBytes, +) -> RvStatus { + take(ctx, || { + // SAFETY: the contract makes `path` and `bytes` valid for their + // lengths during this call. + let (path, contents) = unsafe { (PATH.read(path)?, BYTES.bytes(bytes.ptr, bytes.len)?) }; + Ok(VfsOp::Append { path, contents }) + }) +} + +/// Takes a request to make a directory at `path`; returns RV_OK or +/// RV_ERR_MISUSE. +pub(crate) unsafe extern "C" fn file_mkdir( + ctx: *mut c_void, + path: RvStr, + recursive: u32, +) -> RvStatus { + take(ctx, || { + // SAFETY: the contract makes `path` valid for its length during + // this call. + let path = unsafe { PATH.read(path) }?; + let recursive = flag(recursive)?; + Ok(VfsOp::Mkdir { path, recursive }) + }) +} + +/// Takes a request to remove the file or the directory at `path`; +/// returns RV_OK or RV_ERR_MISUSE. +pub(crate) unsafe extern "C" fn file_remove( + ctx: *mut c_void, + path: RvStr, + recursive: u32, +) -> RvStatus { + take(ctx, || { + // SAFETY: the contract makes `path` valid for its length during + // this call. + let path = unsafe { PATH.read(path) }?; + let recursive = flag(recursive)?; + Ok(VfsOp::Remove { path, recursive }) + }) +} + +/// Takes a request to move the file or the directory at `from` to `to`; +/// returns RV_OK or RV_ERR_MISUSE. +pub(crate) unsafe extern "C" fn file_rename(ctx: *mut c_void, from: RvStr, to: RvStr) -> RvStatus { + take(ctx, || { + // SAFETY: the contract makes both paths valid for their lengths + // during this call. + let (from, to) = unsafe { (FROM.read(from)?, TO.read(to)?) }; + Ok(VfsOp::Rename { from, to }) + }) +} + +/// Takes a request to copy the file at `from` to `to`; returns RV_OK or +/// RV_ERR_MISUSE. +pub(crate) unsafe extern "C" fn file_copy(ctx: *mut c_void, from: RvStr, to: RvStr) -> RvStatus { + take(ctx, || { + // SAFETY: the contract makes both paths valid for their lengths + // during this call. + let (from, to) = unsafe { (FROM.read(from)?, TO.read(to)?) }; + Ok(VfsOp::Copy { from, to }) + }) +} + +/// `recursive` as the contract spells it: 0 for off and 1 for on. +fn flag(recursive: u32) -> Result { + match recursive { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(RECURSIVE), + } +} + +/// What a request asks for: the one field its answer holds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Wanted { + /// What a path names, which `file_stat` asks for. + Stat, + /// The entries of a directory, which `file_list` asks for. + Entries, + /// At most this many bytes of a file, which `file_read` asks for. + Bytes(u64), + /// Nothing but a status, which a function that changes the files + /// asks for. + Nothing, +} + +impl Wanted { + /// What the request for `op` asks for. + pub(crate) fn by(op: &VfsOp) -> Wanted { + match op { + VfsOp::Stat { .. } => Wanted::Stat, + VfsOp::List { .. } => Wanted::Entries, + VfsOp::ReadRange { len, .. } => Wanted::Bytes(*len), + // A change asks for nothing but a status: the six changes of + // the file functions, and `StrReplace` and `Delete`, which no + // file function asks for. No file function asks for a `Read`, + // a `ReadNumbered`, a `Glob`, or an `Exists` either. An answer + // has no field for what one of them returns, so `Answer::to` + // makes a failed answer of its outcome. + VfsOp::Write { .. } + | VfsOp::Append { .. } + | VfsOp::Mkdir { .. } + | VfsOp::Remove { .. } + | VfsOp::Rename { .. } + | VfsOp::Copy { .. } + | VfsOp::StrReplace { .. } + | VfsOp::Delete { .. } + | VfsOp::Read { .. } + | VfsOp::ReadNumbered { .. } + | VfsOp::Glob { .. } + | VfsOp::Exists { .. } => Wanted::Nothing, + } + } +} + +/// The answer to one request, with the memory it points into. A call +/// makes it from the outcome of its request and keeps it until the step +/// that receives it has returned. +pub(crate) struct Answer { + /// What a path names, or zero when the request asked for no stat. + stat: RvStat, /// The bytes of a read. bytes: Vec, /// The names the entries of a listing point at, kept only so that @@ -41,29 +247,67 @@ pub(crate) struct Lent { _names: Vec, /// The entries of a listing. entries: Vec, - /// The text of a failure. - text: String, + /// The kind and the text of the failure, when the operation failed. + error: Option<(RvFileErrorKind, String)>, } -impl Lent { - /// Keeps `bytes` and returns them as the contract hands them back. - fn bytes(&mut self, bytes: Vec) -> RvBytes { - *self = Lent { - bytes, - ..Lent::default() - }; - RvBytes { - ptr: self.bytes.as_ptr(), - len: self.bytes.len(), +// SAFETY: the pointers in `entries` lead into the heap allocations of +// `_names`, which this value owns and never changes, so the value moves +// to another thread whole. +unsafe impl Send for Answer {} + +impl Answer { + /// The answer to a request that asked for `wanted` and whose + /// operation ended as `outcome`: the one thing the request asked for, + /// nothing for a change, or the failure. An outcome of another kind + /// than the request asked for is a failure too, so a step never + /// reads a field the operation did not fill. + pub(crate) fn to(wanted: Wanted, outcome: Result) -> Answer { + match (wanted, outcome) { + (_, Err(error)) => Answer::failed(&error), + (Wanted::Stat, Ok(VfsOutcome::Stat(stat))) => Answer { + stat: raw_stat(&stat), + ..Answer::empty() + }, + (Wanted::Entries, Ok(VfsOutcome::Entries(entries))) => Answer::of_entries(entries), + (Wanted::Bytes(most), Ok(VfsOutcome::Bytes(mut bytes))) => { + // The contract promises at most `len` bytes, whatever the + // backend returned. + bytes.truncate(usize::try_from(most).unwrap_or(usize::MAX)); + Answer { + bytes, + ..Answer::empty() + } + } + (Wanted::Nothing, Ok(VfsOutcome::Unit)) => Answer::empty(), + (_, Ok(_)) => Answer::failed(&VfsError::Backend { + message: "the file operation returned something other than what was asked for" + .to_owned(), + }), } } - /// Keeps `entries` and returns them as the contract hands them back. - fn entries(&mut self, entries: Vec) -> RvDirEntries { + /// An answer that holds nothing. + fn empty() -> Answer { + Answer { + stat: RvStat { + file_type: 0, + reserved: 0, + len: 0, + }, + bytes: Vec::new(), + _names: Vec::new(), + entries: Vec::new(), + error: None, + } + } + + /// The answer that holds `entries`. + fn of_entries(entries: Vec) -> Answer { let stats: Vec = entries.iter().map(|entry| raw_stat(&entry.stat)).collect(); let names: Vec = entries.into_iter().map(|entry| entry.name).collect(); // Each name's bytes sit in its own allocation, so moving `names` - // into `self` leaves the pointers below valid. + // into the answer leaves the pointers below valid. let entries = names .iter() .zip(stats) @@ -72,35 +316,76 @@ impl Lent { stat, }) .collect(); - *self = Lent { + Answer { _names: names, entries, - ..Lent::default() - }; - RvDirEntries { - ptr: self.entries.as_ptr(), - len: self.entries.len(), + ..Answer::empty() } } - /// Keeps the text of `error` and returns the failure as the contract - /// hands it back. - fn error(&mut self, error: &VfsError) -> RvFileError { - let (kind, text) = describe(error); - *self = Lent { - text, - ..Lent::default() + /// The answer of an operation that failed with `error`. + fn failed(error: &VfsError) -> Answer { + Answer { + error: Some(describe(error)), + ..Answer::empty() + } + } + + /// The answer as the contract lends it to a step: the one field the + /// request asked for, or `error`, and every other field zero. It + /// points into this value, which the caller keeps until the step has + /// returned. + pub(crate) fn raw(&self) -> RvFileAnswer { + let (status, error) = match &self.error { + Some((kind, text)) => ( + RV_ERR_FILE, + RvFileError { + kind: *kind, + reserved: 0, + text: rv_str(text), + }, + ), + None => ( + RV_OK, + RvFileError { + kind: 0, + reserved: 0, + text: RvStr { + ptr: ptr::null(), + len: 0, + }, + }, + ), }; - RvFileError { - kind, - reserved: 0, - text: rv_str(&self.text), + RvFileAnswer { + size: size_field::(), + status, + stat: self.stat, + entries: RvDirEntries { + ptr: first(&self.entries), + len: self.entries.len(), + }, + bytes: RvBytes { + ptr: first(&self.bytes), + len: self.bytes.len(), + }, + error, } } } -/// `stat` as the contract hands it back. A kind of node the contract -/// does not name is `RV_FILE_TYPE_OTHER`. +/// Where `values` begin, or NULL when there are none, so that a field +/// with nothing in it is zero. +fn first(values: &[T]) -> *const T { + if values.is_empty() { + ptr::null() + } else { + values.as_ptr() + } +} + +/// `stat` as an answer holds it. A kind of node the contract does not +/// name is `RV_FILE_TYPE_OTHER`. fn raw_stat(stat: &Stat) -> RvStat { RvStat { file_type: match stat.file_type { @@ -116,25 +401,25 @@ fn raw_stat(stat: &Stat) -> RvStat { /// The contract's kind for `error`, and the text a model reads. /// -/// A missing path, a path that is not a directory, and a path that is one -/// keep the error's own sentence. A policy denial carries the policy's -/// own reason. A conflict names the path and not the two identities that -/// claimed it, which mean nothing outside the Harness. An I/O failure -/// carries the kind of the failure and the system's description. Of the -/// errors the contract has no kind for, a refused path carries the rule -/// it broke, and every other one its sentence without the leading label. +/// A missing path, a path that is there already, a path that is not a +/// directory, a path that is one, and a directory that has entries keep +/// the error's own sentence. A policy denial carries the policy's own +/// reason. An I/O failure carries the kind of the failure and the +/// system's description. Of the errors the contract has no kind for, a +/// refused path carries the rule it broke, and every other one its +/// sentence without the leading label. No error reads as +/// `RV_FILE_ERROR_CONFLICT`: the files a tool call reaches check no +/// conflict. fn describe(error: &VfsError) -> (RvFileErrorKind, String) { match error { VfsError::NotFound { .. } => (RV_FILE_ERROR_NOT_FOUND, error.to_string()), + VfsError::AlreadyExists { .. } => (RV_FILE_ERROR_ALREADY_EXISTS, error.to_string()), VfsError::NotADirectory { .. } => (RV_FILE_ERROR_NOT_A_DIRECTORY, error.to_string()), VfsError::IsADirectory { .. } => (RV_FILE_ERROR_IS_A_DIRECTORY, error.to_string()), + VfsError::DirectoryNotEmpty { .. } => (RV_FILE_ERROR_NOT_EMPTY, error.to_string()), VfsError::PermissionDenied { reason, .. } | VfsError::ApprovalRequired { reason, .. } => { (RV_FILE_ERROR_DENIED, reason.clone()) } - VfsError::Conflict { path, .. } => ( - RV_FILE_ERROR_CONFLICT, - format!("another tool call of the run changed {path}, or is changing it"), - ), VfsError::Io { kind, message, .. } => (RV_FILE_ERROR_IO, format!("{kind}: {message}")), // The path is quoted in the sentence and may hold `": "` itself, // so the rule is taken from the error, not cut out of the text. @@ -147,18 +432,6 @@ fn describe(error: &VfsError) -> (RvFileErrorKind, String) { } } -/// Runs one operation of a tool call's access, and turns a panic inside -/// it into a failure. The operation runs under a C function the Plugin -/// called, which a panic must not unwind through, and the policy and the -/// backend behind an access are the Host's own code. -fn contained(operation: impl FnOnce() -> Result) -> Result { - catch_unwind(AssertUnwindSafe(operation)).unwrap_or_else(|_| { - Err(VfsError::Backend { - message: "the file operation panicked".to_owned(), - }) - }) -} - /// `text` without its leading label: the text after the first `": "`, or /// the whole text when it has none. fn without_label(text: &str) -> String { @@ -166,114 +439,3 @@ fn without_label(text: &str) -> String { .map_or(text, |(_, rest)| rest) .to_owned() } - -/// Says what `path` names; returns RV_OK with `stat` filled, RV_ERR_FILE -/// with `error` filled, or RV_ERR_MISUSE. -pub(crate) unsafe extern "C" fn file_stat( - ctx: *mut c_void, - path: RvStr, - stat: *mut RvStat, - error: *mut RvFileError, -) -> RvStatus { - answer_file(ctx, |access, lent| { - let stat = STAT.check(stat)?; - let error = STAT_ERROR.check(error)?; - // SAFETY: the contract makes `path` valid for its length during - // this call. - let path = unsafe { PATH.read(path) }?; - Ok(match contained(|| access.stat(&path)) { - Ok(found) => { - *lent = Lent::default(); - // SAFETY: `stat` is not NULL and is aligned, and the - // contract makes it a struct the Plugin owns. - unsafe { stat.write(raw_stat(&found)) }; - RV_OK - } - // SAFETY: as above, for `error`. - Err(failure) => unsafe { fail(lent, error, &failure) }, - }) - }) -} - -/// Lists the directory at `path`; returns RV_OK with `entries` filled, -/// RV_ERR_FILE with `error` filled, or RV_ERR_MISUSE. -pub(crate) unsafe extern "C" fn file_list( - ctx: *mut c_void, - path: RvStr, - entries: *mut RvDirEntries, - error: *mut RvFileError, -) -> RvStatus { - answer_file(ctx, |access, lent| { - let entries = ENTRIES.check(entries)?; - let error = LIST_ERROR.check(error)?; - // SAFETY: the contract makes `path` valid for its length during - // this call. - let path = unsafe { PATH.read(path) }?; - Ok(match contained(|| access.list(&path)) { - Ok(listed) => { - let listed = lent.entries(listed); - // SAFETY: `entries` is not NULL and is aligned, and the - // contract makes it a struct the Plugin owns. - unsafe { entries.write(listed) }; - RV_OK - } - // SAFETY: as above, for `error`. - Err(failure) => unsafe { fail(lent, error, &failure) }, - }) - }) -} - -/// Reads up to `len` bytes of the file at `path` from the byte `offset` -/// on; returns RV_OK with `bytes` filled, RV_ERR_FILE with `error` filled, -/// or RV_ERR_MISUSE. -pub(crate) unsafe extern "C" fn file_read( - ctx: *mut c_void, - path: RvStr, - offset: u64, - len: u64, - bytes: *mut RvBytes, - error: *mut RvFileError, -) -> RvStatus { - answer_file(ctx, |access, lent| { - let bytes = BYTES.check(bytes)?; - let error = READ_ERROR.check(error)?; - // SAFETY: the contract makes `path` valid for its length during - // this call. - let path = unsafe { PATH.read(path) }?; - // No read returns more than memory can hold, and a backend that - // slices a whole file refuses a length it cannot address. - let len = usize::try_from(len).unwrap_or(usize::MAX); - // A real file cannot be positioned past this offset, and no file - // is that long, so the read finds the end and returns no bytes. - let offset = offset.min(i64::MAX.unsigned_abs()); - let wanted = u64::try_from(len).unwrap_or(u64::MAX); - Ok( - match contained(|| access.read_range(&path, offset, wanted)) { - Ok(mut read) => { - // The contract promises at most `len` bytes, whatever - // the backend returned. - read.truncate(len); - let read = lent.bytes(read); - // SAFETY: `bytes` is not NULL and is aligned, and the - // contract makes it a struct the Plugin owns. - unsafe { bytes.write(read) }; - RV_OK - } - // SAFETY: as above, for `error`. - Err(failure) => unsafe { fail(lent, error, &failure) }, - }, - ) - }) -} - -/// Hands `failure` back through `error`, with its text kept in `lent`, -/// and returns RV_ERR_FILE. -/// -/// # Safety -/// `error` must be valid for a write of an `RvFileError`. -unsafe fn fail(lent: &mut Lent, error: *mut RvFileError, failure: &VfsError) -> RvStatus { - let raw = lent.error(failure); - // SAFETY: the caller makes `error` valid for the write. - unsafe { error.write(raw) }; - RV_ERR_FILE -} diff --git a/crates/harness-internal/plugin-loader/src/host-tests.rs b/crates/harness-internal/plugin-loader/src/host-tests.rs index 3faf08761..7efd96800 100644 --- a/crates/harness-internal/plugin-loader/src/host-tests.rs +++ b/crates/harness-internal/plugin-loader/src/host-tests.rs @@ -465,9 +465,9 @@ fn the_load_call_carries_the_contract_fields() { } #[test] -fn the_table_reports_version_0_4_in_16_bytes() { +fn the_table_reports_version_0_5_in_16_bytes() { assert_eq!(HOST.api_major, 0); - assert_eq!(HOST.api_minor, 4); + assert_eq!(HOST.api_minor, 5); assert_eq!(HOST.reserved, 0); assert_eq!(HOST.size, 16); assert_eq!(usize::try_from(HOST.size), Ok(size_of::())); diff --git a/crates/harness-internal/plugin-loader/src/lib.rs b/crates/harness-internal/plugin-loader/src/lib.rs index 429de3ac3..d13bb8db4 100644 --- a/crates/harness-internal/plugin-loader/src/lib.rs +++ b/crates/harness-internal/plugin-loader/src/lib.rs @@ -8,7 +8,7 @@ //! ## Invariants //! //! - May depend on: `runvil-plugin` among workspace crates, and -//! `runvil-plugin-sys`, the C contract at version 0.4, from its own +//! `runvil-plugin-sys`, the C contract at version 0.5, from its own //! repository. //! - Every entry point gets the same calls, whether it came from a library //! or was linked in. The only difference is that a library is opened @@ -24,16 +24,22 @@ //! borrows its instance, so the delete comes after every call of the //! instance has ended. An `RV_PACK_NEW` that fails gets no //! `RV_PACK_DELETE`. -//! - A tool call sends one `RV_TOOL_CALL` each time its future is polled, -//! and a step that returns `RV_PENDING` wakes the future's own task. A -//! call whose last step returned `RV_PENDING` gets one `RV_TOOL_CANCEL` -//! when it fails or its future is dropped, and no other call does. +//! - Each poll of a tool call's future sends one `RV_TOOL_CALL` at most, +//! and a step that returns `RV_PENDING` with no file request wakes the +//! future's own task. A call whose last step returned `RV_PENDING` +//! gets one `RV_TOOL_CANCEL` when it fails or its future is dropped, +//! and no other call does. //! - The functions in a call's arguments struct answer only that call's //! `ctx`, on its thread, while it runs. -//! - A step's file functions read through the `Access` of the step's tool -//! call and nothing else, so the run's mode and policy apply to every -//! read. What a file function hands back is freed by the step's next -//! file function, or when the step returns. +//! - A step's file function takes one request and fills nothing. The +//! loader asks the files of the call's `ToolContext` for the operation +//! and reaches the run's files in no other way, so the Harness performs +//! and records every file operation of a library Pack's tool, and the +//! run's mode and policy apply to each one. +//! - A call whose request waits for its outcome gets no step and wakes +//! nothing. The step after the outcome is lent the answer, which stays +//! valid until that step returns, and no other step is lent one. The +//! request of a step that ends its call is not performed. //! - Each `RV_PLUGIN_LOAD` gets its own `ctx`, a loader-owned allocation //! freed when a rejected load returns or after its Plugin's //! `RV_PLUGIN_UNLOAD` returns. The `ctx` of every other call is freed diff --git a/crates/harness-internal/plugin-loader/src/pack-tests.rs b/crates/harness-internal/plugin-loader/src/pack-tests.rs index 6866ccddb..ef4b5c463 100644 --- a/crates/harness-internal/plugin-loader/src/pack-tests.rs +++ b/crates/harness-internal/plugin-loader/src/pack-tests.rs @@ -90,10 +90,51 @@ fn an_instance_holds_the_tools_the_plugin_added_under_the_pack_id() { ); assert!( tools.iter().all(|tool| !tool.structured_output + && !tool.survives_stop + && !tool.describes_tools + && !tool.model_only_call + && !tool.changes_files), + "a tool that sets no flag of the contract has every flag clear" + ); +} + +#[test] +fn a_tool_whose_flags_say_so_changes_files_and_no_other_tool_of_the_instance_does() { + pack_fake!( + plug, + {}, + new: |call| { + set_pack_run(call, instance(PACK_TAG)); + add_tool(call, "read"); + add_tool_info( + call, + &RvToolInfo { + flags: RV_TOOL_CHANGES_FILES, + ..tool_info("write") + }, + ) + }, + step: |_call| RV_ERR_UNSUPPORTED + ); + let pack = made(plug::entry); + let flagged: Vec<(String, bool)> = pack + .tools() + .iter() + .map(|tool| (tool.id.to_string(), tool.changes_files)) + .collect(); + assert_eq!( + flagged, + [ + ("fake/read".to_owned(), false), + ("fake/write".to_owned(), true) + ] + ); + assert!( + pack.tools().iter().all(|tool| !tool.structured_output && !tool.survives_stop && !tool.describes_tools && !tool.model_only_call), - "the contract carries no tool flags" + "the contract has no other tool flag" ); } @@ -390,17 +431,41 @@ misuse_test!( ); misuse_test!( - a_nonzero_reserved_field_in_a_tool_info_is_misuse, - "the info's reserved field is not zero", + a_flag_bit_that_is_no_tool_constant_is_misuse, + "the tool's flags set a bit that is not one of the RV_TOOL constants", + |call| add_tool_info( + call, + &RvToolInfo { + flags: 2, + ..tool_info("fetch") + } + ) +); + +misuse_test!( + a_flag_bit_that_is_no_tool_constant_is_misuse_beside_one_that_is, + "the tool's flags set a bit that is not one of the RV_TOOL constants", |call| add_tool_info( call, &RvToolInfo { - reserved: 1, + flags: RV_TOOL_CHANGES_FILES | 0x8000_0000, ..tool_info("fetch") } ) ); +misuse_test!( + the_flags_of_a_tool_are_checked_before_its_name, + "the tool's flags set a bit that is not one of the RV_TOOL constants", + |call| add_tool_info( + call, + &RvToolInfo { + flags: 2, + ..tool_info("web/fetch") + } + ) +); + misuse_test!( a_null_tool_info_is_misuse, "add_tool was handed a NULL info", diff --git a/crates/harness-internal/plugin-loader/src/pack.rs b/crates/harness-internal/plugin-loader/src/pack.rs index 4cbda2d23..3f9030998 100644 --- a/crates/harness-internal/plugin-loader/src/pack.rs +++ b/crates/harness-internal/plugin-loader/src/pack.rs @@ -12,8 +12,8 @@ use runvil_plugin::{ ToolOutput, make_tool_error, }; use runvil_plugin_sys::{ - RV_MAX_NAME, RV_MAX_TOOL_TEXT, RV_OK, RV_PACK_DELETE, RV_PACK_NEW, RvCall, RvPackNewArgs, - RvStatus, RvStr, RvToolInfo, + RV_MAX_NAME, RV_MAX_TOOL_TEXT, RV_OK, RV_PACK_DELETE, RV_PACK_NEW, RV_TOOL_CHANGES_FILES, + RvCall, RvPackNewArgs, RvStatus, RvStr, RvToolInfo, }; use serde_json::Value; @@ -21,8 +21,7 @@ use crate::call::Steps; use crate::host::{is_segment, new_call}; use crate::plugin::Plugin; use crate::session::{ - self, AddedTool, Field, PACK_NEW, RESERVED, Shape, Verb, answer, field, rv_str, shape, - size_field, + self, AddedTool, Field, PACK_NEW, Shape, Verb, answer, field, rv_str, shape, size_field, }; #[cfg(test)] @@ -77,7 +76,8 @@ impl PackFactory for EntryFactory { } } -/// The catalog entry of the tool `added` under the Pack id `id`. +/// The catalog entry of the tool `added` under the Pack id `id`. The +/// entry says that the tool changes files when the tool's flags did. fn tool_info(id: &PackId, added: AddedTool) -> Result { let tool_id = ToolId::parse(&format!("{id}/{}", added.name)).map_err(|error| { make_tool_error( @@ -89,7 +89,8 @@ fn tool_info(id: &PackId, added: AddedTool) -> Result { ) .with_cause(&error) })?; - let info = ToolInfo::new(tool_id, added.description, added.parameters); + let info = ToolInfo::new(tool_id, added.description, added.parameters) + .changes_files(added.changes_files); Ok(match added.wire_name { Some(wire_name) => info.wire_name(wire_name), None => info, @@ -156,7 +157,7 @@ impl Pack for EntryPack { cx: ToolContext<'a>, args: Value, ) -> BoxFuture<'a, Result> { - Box::pin(Steps::new(self, cx.tool().name(), &args, cx.access())) + Box::pin(Steps::new(self, cx.tool().name(), &args, cx.files())) } } @@ -174,8 +175,8 @@ pub(crate) unsafe extern "C" fn add_tool(ctx: *mut c_void, info: *const RvToolIn // SAFETY: the contract makes `info` point at a readable // `RvToolInfo` during this call. let info = unsafe { TOOL_INFO.read(info) }?; - if info.reserved != 0 { - return Err(RESERVED); + if info.flags & !RV_TOOL_CHANGES_FILES != 0 { + return Err("the tool's flags set a bit that is not one of the RV_TOOL constants"); } // SAFETY: the contract makes each string valid for its length. let name = unsafe { TOOL_NAME.read(info.name) }?; @@ -209,6 +210,7 @@ pub(crate) unsafe extern "C" fn add_tool(ctx: *mut c_void, info: *const RvToolIn wire_name: (!wire_name.is_empty()).then_some(wire_name), description, parameters, + changes_files: info.flags & RV_TOOL_CHANGES_FILES != 0, }); Ok(RV_OK) }) diff --git a/crates/harness-internal/plugin-loader/src/session-tests.rs b/crates/harness-internal/plugin-loader/src/session-tests.rs index 38e241ce5..9c9467307 100644 --- a/crates/harness-internal/plugin-loader/src/session-tests.rs +++ b/crates/harness-internal/plugin-loader/src/session-tests.rs @@ -1,21 +1,19 @@ //! Tests for the checks every function a Plugin calls back begins with: -//! what a string field and a struct pointer are refused for, and which -//! call a function answers. +//! what a string field, a run of bytes, and a struct pointer are refused +//! for, and which call a function answers. use std::ffi::c_void; use std::ptr; -use runvil_plugin_sys::{RV_ERR_MISUSE, RV_ERR_REFUSED, RV_OK, RvPluginInfo, RvStat, RvStr}; +use runvil_plugin_sys::{RV_ERR_MISUSE, RV_ERR_REFUSED, RV_OK, RvPluginInfo, RvStr}; use super::{ - Field, LOAD, Out, PACK_NEW, Shape, TOOL_CALL, Verb, answer, answer_file, closed, field, open, - open_step, out, refuse, rv_str, shape, + Field, LOAD, PACK_NEW, Shape, TOOL_CALL, Verb, answer, closed, field, open, refuse, rv_str, + shape, }; -use crate::fakes::test_call; const WORD: Field = field!("the word", 4); const INFO: Shape = shape!("describe_plugin", "info", "RvPluginInfo"); -const STAT: Out = out!("file_stat", "stat"); fn bytes(bytes: &[u8]) -> RvStr { RvStr { @@ -58,6 +56,26 @@ fn a_field_names_each_way_a_string_breaks_its_rules() { assert_eq!(not_utf8, Err("the word is not UTF-8")); } +#[test] +fn a_field_copies_a_run_of_bytes_within_its_bound_whatever_the_bytes_are() { + let held = [0xff_u8, 0x00, 0xfe]; + // SAFETY: the pointer leads to a live array of that length. + let copied = unsafe { WORD.bytes(held.as_ptr(), held.len()) }; + assert_eq!(copied, Ok(held.to_vec())); + // SAFETY: a zero length reads no byte. + let empty = unsafe { WORD.bytes(ptr::null(), 0) }; + assert_eq!(empty, Ok(Vec::new())); + // SAFETY: the length is refused before any byte is read. + let too_long = unsafe { WORD.bytes(held.as_ptr(), 5) }; + assert_eq!(too_long, Err("the word is longer than 4 bytes")); + // SAFETY: a NULL pointer is refused before any byte is read. + let null = unsafe { WORD.bytes(ptr::null(), 2) }; + assert_eq!( + null, + Err("the word has a NULL pointer and a nonzero length") + ); +} + #[test] fn a_shape_names_each_way_a_struct_is_handed_over_wrongly() { let whole = RvPluginInfo { @@ -87,45 +105,6 @@ fn a_shape_names_each_way_a_struct_is_handed_over_wrongly() { ); } -#[test] -fn an_out_names_each_way_a_struct_to_fill_is_handed_over_wrongly() { - let mut stat = RvStat { - file_type: 0, - reserved: 0, - len: 0, - }; - let at = &raw mut stat; - assert_eq!(STAT.check(at), Ok(at)); - assert_eq!( - STAT.check(ptr::null_mut::()), - Err("file_stat was handed a NULL stat pointer") - ); - assert_eq!( - STAT.check(at.wrapping_byte_add(1)), - Err("file_stat was handed a misaligned stat pointer") - ); -} - -#[test] -fn a_file_function_gets_the_access_of_the_open_step_and_no_other_call_has_one() { - let call = test_call("tool"); - let access = call.context().access(); - let (status, session) = open_step(access, |ctx| { - answer_file(ctx, |files, _lent| { - assert!(ptr::eq(files, access), "the step's own access"); - Ok(RV_OK) - }) - }); - assert_eq!(status, RV_OK); - assert_eq!(session.misuse, None); - - for verb in [Verb::Load, Verb::PackNew, Verb::ToolCall] { - let (status, session) = open(verb, |ctx| answer_file(ctx, |_, _| Ok(RV_OK))); - assert_eq!(status, RV_ERR_MISUSE, "{verb:?}"); - assert_eq!(session.misuse, Some(TOOL_CALL.wrong_call), "{verb:?}"); - } -} - #[test] fn a_function_answers_only_the_open_call_of_its_own_verb_through_its_own_ctx() { for (verb, family, other) in [ diff --git a/crates/harness-internal/plugin-loader/src/session.rs b/crates/harness-internal/plugin-loader/src/session.rs index ccc5eaaa3..e46356120 100644 --- a/crates/harness-internal/plugin-loader/src/session.rs +++ b/crates/harness-internal/plugin-loader/src/session.rs @@ -5,16 +5,13 @@ use std::cell::Cell; use std::ffi::c_void; use std::mem::size_of; -use std::panic::{RefUnwindSafe, UnwindSafe}; use std::ptr::{self, NonNull}; use std::slice; -use runvil_plugin::{Access, Error, ToolOutput}; +use runvil_plugin::{Error, ToolOutput, VfsOp}; use runvil_plugin_sys::{RV_ERR_MISUSE, RV_ERR_REFUSED, RV_MAX_LINE, RvStatus, RvStr}; use serde_json::Value; -use crate::file::Lent; - #[cfg(test)] #[path = "session-tests.rs"] mod tests; @@ -55,28 +52,11 @@ pub(crate) struct Session { pub(crate) tools: Vec, /// The result a step finished its call with. pub(crate) result: Option>, - /// The files a step's file functions read: the access of the step's - /// tool call. `open_step` sets it while the step runs, and it is - /// `None` in every other call and once the step has returned. - access: Option, - /// What a step's last file function handed the Plugin, which stays - /// valid until the next one, or until the step returns. - lent: Lent, + /// The file request a step made: the one operation a file function + /// of the step asked for. + pub(crate) request: Option, } -/// The access of the tool call whose step is open. A pointer, not a -/// borrow, because a session is a `'static` allocation that its `ctx` -/// names. -#[derive(Clone, Copy)] -struct StepAccess(NonNull); - -// A loaded Plugin owns the session of its load, which never names an -// access, and a step's session names one only while the step runs. So the -// pointer gives nobody a view of an `Access` after a panic, and a `Plugin` -// stays unwind-safe, as it is without this field. -impl UnwindSafe for StepAccess {} -impl RefUnwindSafe for StepAccess {} - /// The name and version a Plugin gave `describe_plugin`, copied. pub(crate) struct Described { pub(crate) name: String, @@ -99,6 +79,8 @@ pub(crate) struct AddedTool { pub(crate) description: String, /// The parsed JSON Schema, an object. pub(crate) parameters: Value, + /// Whether a call of the tool changes files of its run. + pub(crate) changes_files: bool, } /// Sends one call of `verb` with a fresh session open on this thread. @@ -108,24 +90,6 @@ pub(crate) struct AddedTool { pub(crate) fn open( verb: Verb, send: impl FnOnce(*mut c_void) -> RvStatus, -) -> (RvStatus, Box) { - open_over(verb, None, send) -} - -/// Sends one step of a tool call, as [`open`] sends a call, with `access` -/// as the files its file functions read. -pub(crate) fn open_step( - access: &Access, - send: impl FnOnce(*mut c_void) -> RvStatus, -) -> (RvStatus, Box) { - open_over(Verb::ToolCall, Some(access), send) -} - -/// Sends one call of `verb` over `access`; see [`open`]. -fn open_over( - verb: Verb, - access: Option<&Access>, - send: impl FnOnce(*mut c_void) -> RvStatus, ) -> (RvStatus, Box) { let session = NonNull::from(Box::leak(Box::new(Session { verb, @@ -135,8 +99,7 @@ fn open_over( packs: Vec::new(), tools: Vec::new(), result: None, - access: access.map(|access| StepAccess(NonNull::from(access))), - lent: Lent::default(), + request: None, }))); // A call inside a call, on one thread, keeps the outer session. let outer = OPEN.with(|cell| cell.replace(Some(session))); @@ -147,12 +110,7 @@ fn open_over( OPEN.with(|cell| cell.set(outer)); // SAFETY: `session` came from `Box::leak` above, and with the call // closed no slot holds it, so this is its only owner. - let mut session = unsafe { Box::from_raw(session.as_ptr()) }; - // The borrow of `access` ends with this function, so the session must - // not name it past here, and what was lent is no longer valid. - session.access = None; - session.lent = Lent::default(); - (status, session) + (status, unsafe { Box::from_raw(session.as_ptr()) }) } /// Sends a call that has no arguments, with no session open on this @@ -238,23 +196,6 @@ pub(crate) fn answer( .unwrap_or(RV_ERR_MISUSE) } -/// Answers a file function, as [`answer`] answers a function of -/// [`TOOL_CALL`]. `f` gets the files of the open step and the place that -/// keeps what the function hands back. -pub(crate) fn answer_file( - ctx: *mut c_void, - f: impl FnOnce(&Access, &mut Lent) -> Result, -) -> RvStatus { - answer(ctx, &TOOL_CALL, |session| { - // Only `open_step` opens a step, and it always names an access. - let StepAccess(access) = session.access.ok_or(TOOL_CALL.wrong_call)?; - // SAFETY: `open_step` set the pointer from a reference that - // outlives the step and clears it when the step returns, and - // `answer` runs this only while that step is open on this thread. - f(unsafe { access.as_ref() }, &mut session.lent) - }) -} - /// Records the Plugin's refusal of the open call of `family`, validated; /// returns RV_ERR_REFUSED, or RV_ERR_MISUSE for a `ctx` it doesn't answer /// or a second refusal. @@ -270,7 +211,8 @@ pub(crate) fn refuse(ctx: *mut c_void, family: &Family, reason: RvStr) -> RvStat }) } -/// One string's bound, and what each way of breaking it is called. +/// The bound of one string, or of one run of bytes, and what each way of +/// breaking it is called. pub(crate) struct Field { cap: usize, too_long: &'static str, @@ -278,9 +220,10 @@ pub(crate) struct Field { not_utf8: &'static str, } -/// The [`Field`] of the string `$what` names, bounded at `$cap` bytes. -/// The number is a literal, because it is part of the text of a misuse. A -/// third argument is the contract's limit, which the number must equal. +/// The [`Field`] of the string, or the run of bytes, `$what` names, +/// bounded at `$cap` bytes. The number is a literal, because it is part +/// of the text of a misuse. A third argument is the contract's limit, +/// which the number must equal. macro_rules! field { ($what:literal, $cap:literal $(, $limit:expr)?) => {{ $(const _: () = assert!($cap == $limit);)? @@ -313,30 +256,39 @@ impl Field { } } - /// Copies the string `s` holds. The length is checked against the cap - /// before any byte is read, and `s.ptr` is read only when `s.len` is + /// Copies the string `s` holds, which must be UTF-8, under the rules + /// of [`Field::bytes`]. + /// + /// # Safety + /// The rule of [`Field::bytes`], for `s.ptr` and `s.len`. + pub(crate) unsafe fn read(&self, s: RvStr) -> Result { + // SAFETY: the caller's guarantee is that of `Field::bytes`. + let bytes = unsafe { self.bytes(s.ptr.cast::(), s.len) }?; + String::from_utf8(bytes).map_err(|_| self.not_utf8) + } + + /// Copies the `len` bytes at `ptr`. The length is checked against the + /// cap before any byte is read, and `ptr` is read only when `len` is /// nonzero, because `slice::from_raw_parts` with a NULL pointer is /// undefined behavior even at length 0. /// /// # Safety - /// When `s.len` is nonzero, `s.ptr` must be valid for reads of `s.len` - /// bytes for the duration of this call. - pub(crate) unsafe fn read(&self, s: RvStr) -> Result { - if s.len > self.cap { + /// When `ptr` is not NULL and `len` is from 1 to the cap, `ptr` must + /// be valid for reads of `len` bytes for the duration of this call. + /// No other `ptr` is read. + pub(crate) unsafe fn bytes(&self, ptr: *const u8, len: usize) -> Result, &'static str> { + if len > self.cap { return Err(self.too_long); } - if s.len == 0 { - return Ok(String::new()); + if len == 0 { + return Ok(Vec::new()); } - if s.ptr.is_null() { + if ptr.is_null() { return Err(self.null_with_length); } - // SAFETY: `s.ptr` is not NULL and `s.len` is at most the cap, and - // the caller guarantees `s.ptr` is valid for `s.len` bytes. - let bytes = unsafe { slice::from_raw_parts(s.ptr.cast::(), s.len) }; - std::str::from_utf8(bytes) - .map(str::to_owned) - .map_err(|_| self.not_utf8) + // SAFETY: `ptr` is not NULL and `len` is at most the cap, and the + // caller guarantees `ptr` is valid for `len` bytes. + Ok(unsafe { slice::from_raw_parts(ptr, len) }.to_vec()) } } @@ -403,44 +355,6 @@ impl Shape { } } -/// One struct a Plugin hands a function to fill, and what each way of -/// handing it over wrongly is called. -pub(crate) struct Out { - null: &'static str, - misaligned: &'static str, -} - -/// The [`Out`] of the struct the function `$function` fills and calls -/// its `$what`. -macro_rules! out { - ($function:literal, $what:literal) => { - $crate::session::Out::new( - concat!($function, " was handed a NULL ", $what, " pointer"), - concat!($function, " was handed a misaligned ", $what, " pointer"), - ) - }; -} -pub(crate) use out; - -impl Out { - /// An out struct with the name of each breach. - pub(crate) const fn new(null: &'static str, misaligned: &'static str) -> Out { - Out { null, misaligned } - } - - /// `at`, once it is known to be a place a `T` can be written to. It - /// reads nothing. - pub(crate) fn check(&self, at: *mut T) -> Result<*mut T, &'static str> { - if at.is_null() { - return Err(self.null); - } - if !at.is_aligned() { - return Err(self.misaligned); - } - Ok(at) - } -} - /// `size_of::()` as the `u32` a contract struct's `size` field holds. #[expect( clippy::cast_possible_truncation, diff --git a/crates/harness-internal/runner/src/effect_loop-tests.rs b/crates/harness-internal/runner/src/effect_loop-tests.rs index 9a86e9cfc..66274b6af 100644 --- a/crates/harness-internal/runner/src/effect_loop-tests.rs +++ b/crates/harness-internal/runner/src/effect_loop-tests.rs @@ -12,11 +12,11 @@ use runvil_engine::effect::{Round, ToolCallOrigin}; use runvil_engine::model::{ Completion, CompletionOptions, Message, ModelBinding, ModelCatalog, ToolSchema, }; -use runvil_engine::vfs::Access; use runvil_engine::{ BoxFuture, Error, ErrorCode, Prompt, Run, RunParams, RunSeed, Step, Timestamp, }; use runvil_engine::{ToolCatalog, ToolId, ToolInfo, ToolOutput}; +use runvil_plugin::ToolFiles; use serde_json::{Value, json}; use super::{STALLED, drive}; @@ -110,9 +110,9 @@ impl ToolPerformer for Unused { &self, _tool: ToolId, _alias: String, - _access: Arc, _origin: ToolCallOrigin, _args: Value, + _files: Arc, ) -> BoxFuture<'static, Result> { unreachable!("the loop never sees the lost run's tool call") } diff --git a/crates/harness-internal/runner/src/effect_loop.rs b/crates/harness-internal/runner/src/effect_loop.rs index 6bb448059..b9eaa7a23 100644 --- a/crates/harness-internal/runner/src/effect_loop.rs +++ b/crates/harness-internal/runner/src/effect_loop.rs @@ -10,25 +10,56 @@ //! future: the loop starts no task, so a run is one future that any //! executor can drive. When an effect's future lands, the loop appends the //! answer's record and resumes the run with it, then steps again. A Vfs -//! effect has no future: it is one operation on a store of the run, which -//! is memory, so the loop performs the operation inline through the -//! effect's view, appends the answer's record, and resumes the run at -//! once. A run has the store it starts with and one more for each prompt -//! an `execute` runs, and the effect's record names its store, so the -//! loop reads the view and never which store it reaches. It resumes every -//! Vfs effect of a step this way and then steps once, so the loop awaits -//! an answer only when it answered nothing inline. Vfs operations from -//! different chains run one at a time in issue order. +//! effect has no future: it is one file operation, and file operations +//! are synchronous by design, so the loop performs the operation inline +//! through the effect's access, appends the answer's record, and resumes +//! the run at once. It resumes every Vfs effect of a step this way and +//! then steps once, so the loop awaits an answer only when it answered +//! nothing inline. Vfs operations run one at a time in issue order, +//! whichever chain or tool call asked for each. +//! +//! The prompt's Lua asks for a Vfs effect with a `store` call, and the +//! operation works on a store of the run. A run has the store it starts +//! with and one more for each prompt an `execute` runs, and the effect's +//! record names its store, so the loop reads the view and never which +//! store it reaches. The loop answers with the operation's whole outcome. +//! +//! A tool call asks through its files, the desk the loop lends it. Each +//! request joins the run's queue of the calls that asked, which the loop +//! reads beside the effects in flight, before and after it polls them. +//! For a request from a call still in flight, the loop reports the +//! operation to the run with `Run::request_file` and steps, and the run +//! issues it as a Vfs effect whose asker is the call. The loop performs +//! it on the run's filesystem through the call's own access, inside a +//! guard that turns a panic of the Host's policy or backend into a +//! backend failure, appends the answer's record, resumes the run with +//! the receipt of the outcome, and hands the whole outcome to the call's +//! desk, which wakes the call. The tool call stays in flight throughout. +//! A request from a call that has landed or was aborted is dropped, and a +//! Vfs effect whose call has gone since it asked is answered `Dropped` +//! unperformed. +//! +//! While the recorder answers at once, the loop performs a request and +//! wakes its call without returning to its executor, so it makes that +//! return itself. After a set number of requests, and before it reports +//! the next one, the run's future wakes itself and returns pending once. +//! Which task the executor polls next is the executor's choice, so the +//! return is one more chance for its other tasks, one that raises a +//! cancel or a stop among them, and no promise. +//! An executor that takes its tasks in turn runs them then, however long +//! a tool call keeps asking. No Vfs effect is outstanding at that point, +//! and the request is still at its desk. //! //! An effect the step withdraws is answered `Dropped` without being //! performed, and a withdrawn effect an earlier step issued is aborted //! and answered `Dropped`, so the step that decides the run leaves the //! loop nothing to perform. //! -//! The loop waits on four things at once: the run's cancel flag, the stop +//! The loop waits on five things at once: the run's cancel flag, the stop //! a Host raises through [`RunControl::stop_round`](crate::RunControl::stop_round), //! the messages a Host sends through -//! [`RunControl::send`](crate::RunControl::send), and the next effect to +//! [`RunControl::send`](crate::RunControl::send), the next file request +//! of a tool call, and the next effect to //! land. A cancel aborts every effect in flight, //! answers each `Dropped`, and steps the run, which reads the same flag, //! to `Done`. A stop aborts every effect in flight except tool calls that @@ -90,18 +121,20 @@ use std::task::{Context, Poll, Waker}; use futures_util::future::{AbortHandle, Abortable, Aborted, FutureExt as _}; use futures_util::stream::{FuturesUnordered, StreamExt as _}; use runvil_engine::cancel::CancelHandle; -use runvil_engine::effect::{Effect, EffectAnswer, EffectId, IssuedEffect}; +use runvil_engine::effect::{Effect, EffectAnswer, EffectId, IssuedEffect, VfsAsker, VfsReceipt}; use runvil_engine::event::Event; use runvil_engine::ids::{Provenance, TaskId}; -use runvil_engine::vfs::{Access, VfsOp, perform_vfs_op}; +use runvil_engine::vfs::{Access, VfsError, VfsOp, VfsOutcome, perform_vfs_op}; use runvil_engine::{BoxFuture, Error, ErrorCode}; use runvil_engine::{Run, Step}; +use runvil_plugin::ToolFiles; use tracing::Instrument as _; use crate::harness::{RunControl, SendQueue, StopSignal}; use crate::performers::{InferenceBroker, Timer, ToolPerformer}; use crate::recorder::{Record, RecordKind, RunId, RunRecorder, make_recorder_error, refused}; use crate::setup::RunSetup; +use crate::tool_files::{Asked, CallDesk}; /// The message of the failure a stalled run ends with. const STALLED: &str = "the effect loop has nothing to await for a pending run"; @@ -167,6 +200,7 @@ pub(crate) async fn drive( recorder: Arc::clone(&setup.recorder), run_id, flights: Flights::new(setup.max_tool_calls_in_flight), + asked: Arc::new(Asked::default()), cancel: control.cancel_handle().clone(), stop: Arc::clone(control.stop_signal()), sends: Arc::clone(control.send_queue()), @@ -191,6 +225,9 @@ struct Driver { /// tool calls waiting for a slot. Dropping the driver drops their futures, so /// nothing a performer holds outlives the drive however it ends. flights: Flights, + /// The tool calls that asked for a file operation, one entry for each + /// request the loop has not taken off its call's desk. + asked: Arc, cancel: CancelHandle, stop: Arc, /// The messages the Host sent that the loop has not delivered yet. @@ -208,10 +245,20 @@ enum Woken { Stop, /// The Host sent these messages, oldest first. Sent(Vec), + /// This tool call asked for a file operation. + Asked(EffectId), /// An effect's future landed with its answer. Landed(EffectId, Provenance, EffectAnswer), } +/// A Vfs effect, which the loop answers inline: who asked for it, the +/// access it runs on, and the operation. +struct InlineVfs { + asker: VfsAsker, + access: Arc, + op: VfsOp, +} + impl Driver { async fn drive(&mut self) -> Result { let early = self.sends.take(); @@ -287,9 +334,8 @@ impl Driver { // answered here; every other kind goes in flight. // The run is not stepped between inline answers: // the whole batch is resumed, then stepped once. - if let Some((access, op)) = self.perform(id, provenance, issued.effect) { - let answer = answer_vfs(access, op); - self.answer(id, provenance, answer).await?; + if let Some(vfs) = self.perform(id, provenance, issued.effect) { + self.answer_vfs(id, provenance, vfs).await?; answered = true; } } @@ -337,7 +383,9 @@ impl Driver { /// Waits for the next answer, then answers every effect already /// landed behind it, or acts on a cancel or a stop as soon as one is - /// raised, or delivers what the Host sent. + /// raised, or delivers what the Host sent, or reports a tool call's + /// file request to the run, after a turn of the executor when the + /// run's queue of requests says one is due. /// /// Before a delivery, a stop raised before the messages were sent is /// handled, so a Host that stops and then sends gets the stop, and @@ -355,6 +403,17 @@ impl Driver { self.answer_landed().await?; self.deliver(texts).await } + Woken::Asked(call) => { + // Every Vfs effect is answered before the loop waits, so + // none is outstanding during the turn, and the request + // stays at its desk. A stop or a cancel raised during the + // turn is handled as one raised while a call asks. + if self.asked.turn_due() { + give_a_turn().await; + } + self.report_request(call); + Ok(()) + } Woken::Landed(id, provenance, answer) => { self.answer(id, &provenance, answer).await?; self.answer_landed().await @@ -362,6 +421,59 @@ impl Driver { } } + /// Reports the oldest file request of tool call `call` to the run, + /// which issues it as a Vfs effect at the next step. A call that has + /// landed or was aborted has no desk, and a request whose future was + /// dropped has left its desk, so neither is reported. + fn report_request(&mut self, call: EffectId) { + let Some(op) = self.flights.desk(call).and_then(|desk| desk.next_request()) else { + return; + }; + self.run.request_file(call, op); + } + + /// Answers one Vfs effect inline: performs the operation through the + /// effect's access, records the answer, and resumes the run with it. + /// + /// The prompt's Lua that asked takes the whole outcome as the answer + /// of its `store` call. A tool call that asked gets the whole outcome + /// at its desk, which wakes it, once the run has the outcome's + /// receipt. When that call has landed or was aborted since it asked, + /// the effect is answered `Dropped` unperformed. + /// + /// The access is this function's own, so it drops when the function + /// returns, and the Harness holds no way to a store between effects. + /// The drop is hygiene only: claims follow happens-before within the + /// run's scope, and the run ends that scope at `Done` however long + /// any access is held. + async fn answer_vfs( + &mut self, + id: EffectId, + provenance: &Provenance, + vfs: InlineVfs, + ) -> Result<(), Error> { + let InlineVfs { asker, access, op } = vfs; + let call = match asker { + VfsAsker::Lua { .. } => { + let answer = EffectAnswer::Vfs(perform_vfs_op(&access, &op)); + return self.answer(id, provenance, answer).await; + } + VfsAsker::Tool { call } => call, + }; + let Some(desk) = self.flights.desk(call).cloned() else { + return self.drop_effect(id, provenance).await; + }; + let outcome = perform_contained(&access, &op); + let receipt = match &outcome { + Ok(outcome) => Ok(VfsReceipt::of(outcome)), + Err(error) => Err(error.clone()), + }; + self.answer(id, provenance, EffectAnswer::VfsReceipt(receipt)) + .await?; + desk.deliver(outcome); + Ok(()) + } + /// Acts on a raised stop: drops whatever is in flight but the tool /// calls that survive stops, which may be nothing, then lowers the /// stop. Returns whether it answered any effect, since each answer @@ -373,16 +485,19 @@ impl Driver { Ok(self.flights.len() < owed) } - /// Waits for the cancel flag, a raised stop, a sent message, or the - /// next effect to land, checked in that order, so a stop drops a round - /// whose answer or message arrived beside it. Every arm is - /// event-driven: the cancel, the stop, and a send wake the loop as they - /// are raised and a future wakes it as it lands, so a fully suspended - /// run costs no wakeups while it waits. + /// Waits for the cancel flag, a raised stop, a sent message, a tool + /// call's file request, or the next effect to land, checked in that + /// order, so a stop drops a round whose answer or message arrived + /// beside it. The requests are checked a second time once the effects + /// in flight are polled, because a tool call asks while it is polled. + /// Every arm is event-driven: the cancel, the stop, a send, and a + /// request wake the loop as they are raised and a future wakes it as + /// it lands, so a fully suspended run costs no wakeups while it waits. async fn wake(&mut self) -> Woken { let mut canceled = self.cancel.canceled(); let stop = &self.stop; let sends = &self.sends; + let asked = &self.asked; let flights = &mut self.flights; poll_fn(|cx| { if Pin::new(&mut canceled).poll(cx).is_ready() { @@ -394,9 +509,13 @@ impl Driver { if let Poll::Ready(texts) = sends.poll_sent(cx) { return Poll::Ready(Woken::Sent(texts)); } - flights - .poll_landed(cx) - .map(|(id, provenance, answer)| Woken::Landed(id, provenance, answer)) + if let Poll::Ready(call) = asked.poll_next(cx) { + return Poll::Ready(Woken::Asked(call)); + } + if let Poll::Ready((id, provenance, answer)) = flights.poll_landed(cx) { + return Poll::Ready(Woken::Landed(id, provenance, answer)); + } + asked.poll_next(cx).map(Woken::Asked) }) .await } @@ -442,15 +561,17 @@ impl Driver { /// flight, where a tool call may wait for a slot, and returns `None`. /// The performer is called inside its future, so a performer that /// panics as it is called is caught like one that panics as it runs. + /// A tool call gets a desk of its own, through which its tool asks + /// for file operations while the call is in flight. /// - /// A Vfs effect goes nowhere and is handed back as the access and - /// operation it carries, for the caller to answer inline. + /// A Vfs effect goes nowhere and is handed back as it is, for the + /// caller to answer inline. fn perform( &mut self, id: EffectId, provenance: &Provenance, effect: Effect, - ) -> Option<(Arc, VfsOp)> { + ) -> Option { let (kind, answer): (Kind, BoxFuture<'static, EffectAnswer>) = match effect { Effect::Chat { binding, @@ -484,25 +605,20 @@ impl Driver { alias, args, origin, - access, survives_stop, } => { let performer = Arc::clone(&self.tool); + let desk = CallDesk::open(id, &self.asked); + let files: Arc = Arc::clone(&desk) as _; let call = async move { - EffectAnswer::ToolCall(performer.call(tool, alias, access, origin, args).await) + let output = performer.call(tool, alias, origin, args, files).await; + EffectAnswer::ToolCall(output) }; self.flights - .queue(id, provenance.clone(), survives_stop, Box::pin(call)); + .queue(id, provenance.clone(), survives_stop, desk, Box::pin(call)); return None; } - // The effect's record, written before this call, names the - // store, and the view reaches no other, so the store's id has - // no use here. - Effect::Vfs { - store: _, - access, - op, - } => return Some((access, op)), + Effect::Vfs { asker, access, op } => return Some(InlineVfs { asker, access, op }), Effect::Timer { duration } => { let timer = Arc::clone(&self.timer); let sleep = async move { @@ -630,12 +746,17 @@ struct InFlight { /// Whether the effect is a tool call that survives stops, which a /// stop leaves alone. survives_stop: bool, + /// The desk of a tool call, through which its tool asks for file + /// operations. It goes when the call lands or is aborted. + desk: Option>, } -/// A tool call waiting for a slot: its performer future, not yet polled. +/// A tool call waiting for a slot: its performer future, not yet polled, +/// and its desk. struct Waiting { provenance: Provenance, survives_stop: bool, + desk: Arc, answer: BoxFuture<'static, EffectAnswer>, } @@ -696,18 +817,19 @@ impl Flights { kind: Kind, answer: BoxFuture<'static, EffectAnswer>, ) { - self.launch(id, provenance, kind, false, answer); + self.launch(id, provenance, kind, false, None, answer); } /// Queues `answer`, the performer future of tool call `id`, under /// `provenance`, then starts waiting calls, lowest id first, while /// fewer than `cap` tool calls are in flight. `survives_stop` marks a - /// call that survives stops. + /// call that survives stops, and `desk` is the call's desk. fn queue( &mut self, id: EffectId, provenance: Provenance, survives_stop: bool, + desk: Arc, answer: BoxFuture<'static, EffectAnswer>, ) { self.waiting.insert( @@ -715,12 +837,19 @@ impl Flights { Waiting { provenance, survives_stop, + desk, answer, }, ); self.fill(); } + /// The desk of tool call `call`, while the call is in flight. A call + /// that waits for a slot has not started, so it has asked for nothing. + fn desk(&self, call: EffectId) -> Option<&Arc> { + self.effects.get(&call)?.desk.as_ref() + } + /// Starts waiting calls, lowest id first, while fewer than `cap` tool /// calls are in flight. fn fill(&mut self) { @@ -738,6 +867,7 @@ impl Flights { call.provenance, Kind::ToolCall, call.survives_stop, + Some(call.desk), call.answer, ); running += 1; @@ -748,13 +878,14 @@ impl Flights { /// `provenance` as `kind`, inside a span that records the effect id, /// the task path, and the task-local sequence, so a run's effects /// trace as a group. `survives_stop` marks a tool call that survives - /// stops. + /// stops, and `desk` is a tool call's desk. fn launch( &mut self, id: EffectId, provenance: Provenance, kind: Kind, survives_stop: bool, + desk: Option>, answer: BoxFuture<'static, EffectAnswer>, ) { let span = tracing::info_span!( @@ -774,6 +905,7 @@ impl Flights { abort, kind, survives_stop, + desk, }, ); } @@ -863,21 +995,35 @@ impl Flights { } } -/// Answers one Vfs effect inline: performs `op` on the effect's store -/// through the view the effect carries and returns the answer to record -/// and resume the run with. -/// -/// The view is this function's own parameter, so it drops when the -/// function returns, and the Harness holds no way to a store between -/// effects. The drop is hygiene only: claims follow happens-before within -/// the run's scope, and the run ends that scope at `Done` however long any -/// access is held. -#[expect( - clippy::needless_pass_by_value, - reason = "the view is taken by value so that it drops with the operation" -)] -fn answer_vfs(access: Arc, op: VfsOp) -> EffectAnswer { - EffectAnswer::Vfs(perform_vfs_op(&access, op)) +/// Returns to the executor once. The future wakes itself before it +/// returns pending, so the executor polls the run again, and the future +/// needs no runtime of its own. Which task the executor polls next is +/// the executor's choice: the return is a chance for its other tasks to +/// run, and no promise that one does before the run is polled again. +async fn give_a_turn() { + let mut given = false; + poll_fn(|cx| { + if given { + return Poll::Ready(()); + } + given = true; + cx.waker().wake_by_ref(); + Poll::Pending + }) + .await; +} + +/// Performs `op`, an operation a tool call asked for, on `access`, and +/// turns a panic inside it into a backend failure. The policy and the +/// backends behind the access of a tool call are the Host's own code, and +/// a panic there must reach the tool as a failed operation, where it +/// would otherwise unwind through the run's future. +fn perform_contained(access: &Access, op: &VfsOp) -> Result { + std::panic::catch_unwind(AssertUnwindSafe(|| access.perform(op))).unwrap_or_else(|_| { + Err(VfsError::Backend { + message: "the file operation panicked".to_owned(), + }) + }) } /// A panic payload's message, when it carries one. diff --git a/crates/harness-internal/runner/src/lib.rs b/crates/harness-internal/runner/src/lib.rs index 90e5dea75..fbc1c798c 100644 --- a/crates/harness-internal/runner/src/lib.rs +++ b/crates/harness-internal/runner/src/lib.rs @@ -32,7 +32,9 @@ //! no task, so any executor can drive a run. Performers must not block //! while polled: one that blocks stalls every other effect of the run, //! and a performer with blocking or CPU-heavy work hands it to the -//! Host's own runtime. The crate's async needs come from `futures-util`, +//! Host's own runtime. A file operation is the exception: it is +//! synchronous by design and holds the run until it returns. The +//! crate's async needs come from `futures-util`, //! and `tokio` and `tokio-util` appear only under `[dev-dependencies]` //! (enforced by the Harness tokio ban in `cargo test -p build-xtask`). //! - The Harness reads each [`RunSetup`] field once per run: the run's @@ -42,6 +44,12 @@ //! of a Vfs effect, through that effect's view. It puts nothing in a //! store before a run and reads nothing from one after: a run takes its //! input as arguments and hands back its result. +//! - A tool call asks for a file operation through the files of its +//! context. The Harness reports each request to the run, performs the +//! Vfs effect the run issues for it on the run's filesystem, records +//! it, and hands the outcome to the call, so the record holds every +//! operation a tool asks for this way. It drops the request of a call +//! that has landed or was aborted. //! - The recorder is written in loop order: a step's events before the //! step's effects are issued, each effect before its performer starts, //! each answer before the run resumes with it, and each sent message's @@ -90,6 +98,7 @@ mod prompt_finders; pub mod recorder; mod recorder_memory; mod setup; +mod tool_files; pub use display_chain::display_chain; pub use harness::{Harness, RunControl, RunReport}; diff --git a/crates/harness-internal/runner/src/packs-tests.rs b/crates/harness-internal/runner/src/packs-tests.rs index 33ed1bd93..17d5a0f01 100644 --- a/crates/harness-internal/runner/src/packs-tests.rs +++ b/crates/harness-internal/runner/src/packs-tests.rs @@ -18,12 +18,12 @@ use runvil_engine::cancel::CancelHandle; use runvil_engine::effect::{ToolCallOrigin, ToolCaller}; use runvil_engine::make_tool_error; use runvil_engine::packs::RosterEntry; -use runvil_engine::vfs::{Origin, VfsRef}; +use runvil_engine::vfs::{VfsError, VfsOp, VfsOutcome}; use runvil_engine::{Error, ErrorCode}; use runvil_engine::{Prompt, Run, RunParams, RunSeed, Step, Timestamp}; use runvil_plugin::{ BoxFuture, HostServices, Pack, PackFactory, PackId, PackInfo, PackRegistry, ServiceKey, - ToolContext, ToolId, ToolInfo, ToolOutput, + ToolContext, ToolFiles, ToolId, ToolInfo, ToolOutput, }; use serde_json::{Value, json}; @@ -683,19 +683,106 @@ fn preludes_follow_the_prompts_declaration_order_and_carry_the_pack_id() { assert_eq!(order, ["third", "second", "first"]); } -/// Calls `tool` through `run` from a script and returns its result. +/// The files of a call whose tool asks for no file operation. +struct NoFiles; + +impl ToolFiles for NoFiles { + fn request(&self, op: VfsOp) -> BoxFuture<'_, Result> { + unreachable!("no tool of this call asks for a file operation: {op:?}") + } +} + +/// Files that note each operation asked of them and answer it at once +/// with `true`. +#[derive(Default)] +struct NotedFiles { + asked: Mutex>, +} + +impl ToolFiles for NotedFiles { + fn request(&self, op: VfsOp) -> BoxFuture<'_, Result> { + self.asked.lock().unwrap().push(op); + Box::pin(async { Ok(VfsOutcome::Bool(true)) }) + } +} + +/// Asks its call's files whether `/probe` exists and answers with what +/// they said. +struct Asks; + +impl Pack for Asks { + fn tools(&self) -> Vec { + vec![listed_tool("asks/probe")] + } + + fn call<'a>( + &'a self, + cx: ToolContext<'a>, + _args: Value, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let probe = VfsOp::Exists { + path: "/probe".to_owned(), + }; + let outcome = cx.files().request(probe).await; + Ok(ToolOutput::trusted(format!("{outcome:?}"))) + }) + } +} + +/// Makes an [`Asks`]. +#[expect( + clippy::unnecessary_wraps, + reason = "the Make signature fixes the return type" +)] +fn asks(_name: &PackId, _config: &Value, _wide: &HostServices) -> Result, Error> { + Ok(Arc::new(Asks)) +} + +/// Calls `tool` through `run` from a script, lending the call no files +/// to ask, and returns its result. async fn call(run: &dyn ToolPerformer, tool: &str) -> Result { - let access = VfsRef::default().acquire(Origin::new("host test")).unwrap(); + call_with(run, tool, Arc::new(NoFiles)).await +} + +/// Calls `tool` through `run` from a script, lending the call `files`, +/// and returns its result. +async fn call_with( + run: &dyn ToolPerformer, + tool: &str, + files: Arc, +) -> Result { let origin = ToolCallOrigin::new("Only", ToolCaller::Script); let tool = ToolId::parse(tool).unwrap(); - run.call( - tool, - "alias".to_owned(), - Arc::new(access), - origin, - json!({}), - ) - .await + run.call(tool, "alias".to_owned(), origin, json!({}), files) + .await +} + +#[tokio::test] +async fn a_call_is_lent_the_files_the_caller_gave_for_it() { + let host = host_with( + &HostServices::new(), + &[(info("runvil/asks"), asks, Value::Null)], + ); + let run = begin(&host, BTreeMap::new(), &prompt("")); + let files = Arc::new(NotedFiles::default()); + + let output = call_with(&run, "asks/probe", Arc::clone(&files) as _) + .await + .expect("the call answers"); + + assert_eq!( + output.text(), + "Ok(Bool(true))", + "the tool got the files' answer" + ); + assert_eq!( + *files.asked.lock().unwrap(), + [VfsOp::Exists { + path: "/probe".to_owned(), + }], + "the tool's request reached the files of its own call" + ); } #[tokio::test] diff --git a/crates/harness-internal/runner/src/packs.rs b/crates/harness-internal/runner/src/packs.rs index 94abd891d..bfb0e319a 100644 --- a/crates/harness-internal/runner/src/packs.rs +++ b/crates/harness-internal/runner/src/packs.rs @@ -51,11 +51,10 @@ use runvil_engine::cancel::CancelHandle; use runvil_engine::effect::{PackToolSets, ToolCallOrigin}; use runvil_engine::model::ModelInfo; use runvil_engine::packs::{PackRef, Prelude, RosterEntry}; -use runvil_engine::vfs::Access; use runvil_engine::{BoxFuture, Error, ErrorCode}; use runvil_engine::{ToolCatalog, ToolId, ToolInfo, ToolOutput, make_tool_error}; use runvil_plugin::{ - Pack, PackFactory, PackId, PackInfo, PackRegistry, RenderContext, ToolContext, + Pack, PackFactory, PackId, PackInfo, PackRegistry, RenderContext, ToolContext, ToolFiles, }; use serde_json::Value; @@ -567,9 +566,9 @@ impl ToolPerformer for RunPacks { &self, tool: ToolId, alias: String, - access: Arc, origin: ToolCallOrigin, args: Value, + files: Arc, ) -> BoxFuture<'static, Result> { let pack = self.usable(&tool.pack()).cloned(); Box::pin(async move { @@ -584,7 +583,7 @@ impl ToolPerformer for RunPacks { format!("tool `{alias}` ({tool}) belongs to no usable Pack of this run"), )); }; - let cx = ToolContext::new(&tool, &access, &origin); + let cx = ToolContext::new(&tool, &origin, files.as_ref()); pack.call(cx, args).await }) } diff --git a/crates/harness-internal/runner/src/performers.rs b/crates/harness-internal/runner/src/performers.rs index c26034ad3..6c16e8aef 100644 --- a/crates/harness-internal/runner/src/performers.rs +++ b/crates/harness-internal/runner/src/performers.rs @@ -17,9 +17,10 @@ //! block while polled: one that blocks stalls every other effect of the //! run, and the run's stop and cancel with them. A performer with blocking //! or CPU-heavy work hands it to the Host's own runtime and awaits the -//! result. A `Vfs` effect has no performer: it is one operation on a -//! store of the run, which is memory, so the loop answers it inline -//! through the Engine's store operation. +//! result. A `Vfs` effect has no performer: it is one file operation, on +//! a store of the run or, for a tool call that asked through its files, +//! on the run's filesystem, and file operations are synchronous by +//! design, so the loop answers it inline. //! //! The runner supplies the tool performer itself: the run's snapshot of //! the Host's Packs, taken at preparation. The Host supplies the @@ -32,9 +33,9 @@ use runvil_engine::effect::{PackToolSets, Round, ToolCallOrigin}; use runvil_engine::model::{ Completion, CompletionOptions, Message, ModelBinding, ModelCatalog, ModelInfo, ToolSchema, }; -use runvil_engine::vfs::Access; use runvil_engine::{BoxFuture, Error, Timestamp}; use runvil_engine::{ToolId, ToolInfo, ToolOutput}; +use runvil_plugin::ToolFiles; use serde_json::Value; /// The model inference the Host supplies to the Harness. @@ -73,8 +74,9 @@ pub trait InferenceBroker: Send + Sync { } /// Performs a `ToolCall` effect: resolves `tool` to an implementation and -/// calls it with `args`, lending it the call's `access` and `origin`. Also -/// performs a `Render` effect: the tools each offered Pack shows the model. +/// calls it with `args`, lending it the call's `origin` and `files`. Also +/// performs a `Render` effect: the tools each offered Pack shows the +/// model. /// /// Each method returns a [`BoxFuture`] for the Harness to poll. The future /// is `Send` and `'static`. It can move between threads, and it owns the @@ -82,17 +84,19 @@ pub trait InferenceBroker: Send + Sync { pub trait ToolPerformer: Send + Sync { /// Calls the tool. `alias` is the tool's wire name for a model call and /// its canonical id for a script call, for the performer's own - /// diagnostics; `tool` is the identity it resolves. `access` is the - /// call's own filesystem access and `origin` says who made the call. - /// The returned future owns both until it finishes, and drops them - /// when it finishes or is dropped. + /// diagnostics; `tool` is the identity it resolves. `origin` says who + /// made the call. `files` is the call's desk at the effect loop, and + /// the tool's one way to the run's files: the tool asks it for each + /// file operation, and the loop performs and records the operation + /// and resolves the request. The returned future owns both until it + /// finishes, and drops them when it finishes or is dropped. fn call( &self, tool: ToolId, alias: String, - access: Arc, origin: ToolCallOrigin, args: Value, + files: Arc, ) -> BoxFuture<'static, Result>; /// Renders the tools the section entry `entry` offered from each Pack diff --git a/crates/harness-internal/runner/src/tool_files-tests.rs b/crates/harness-internal/runner/src/tool_files-tests.rs new file mode 100644 index 000000000..66a93f9c6 --- /dev/null +++ b/crates/harness-internal/runner/src/tool_files-tests.rs @@ -0,0 +1,213 @@ +//! Tests for a tool call's desk: a request joins its desk at its future's +//! first poll and announces its call, an outcome wakes and resolves the +//! oldest request the loop took, a dropped future's request is never +//! reported or has its outcome discarded, and the run's queue wakes the +//! loop. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::task::Wake; + +use runvil_engine::{Prompt, Run, RunParams, RunSeed, Step, Timestamp}; + +use super::*; + +/// Counts the times it is woken. +#[derive(Default)] +struct Woken(AtomicUsize); + +impl Wake for Woken { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +impl Woken { + fn count(&self) -> usize { + self.0.load(Ordering::SeqCst) + } +} + +/// A waker that counts on `woken`. +fn waker_of(woken: &Arc) -> Waker { + Waker::from(Arc::clone(woken)) +} + +/// An effect id to open a desk under. Only a run makes an `EffectId`, so +/// this is the id of the one effect a small run issues. +fn call_id() -> EffectId { + let source = "---\nname: t\ndescription: d\nrunvil: 0\n---\n\n\ + ## Only\n\n```lua\nreturn store.exists('x')\n```\n"; + let prompt = Prompt::parse(&|_: &str| Ok(source.to_owned()), "prompt") + .expect("the fixture prompt parses"); + let params = RunParams::new(RunSeed::from_bytes([7; 32]), Timestamp::UNIX_EPOCH); + let mut run = Run::new(Arc::new(prompt), "{}", params); + let Step::Pending { effects, .. } = run.step() else { + panic!("the fixture run issues one effect"); + }; + let [issued] = effects.as_slice() else { + panic!("the fixture run issues one effect: {effects:?}"); + }; + issued.id +} + +/// A write of `text` to `path`. +fn write(path: &str, text: &str) -> VfsOp { + VfsOp::Write { + path: path.to_owned(), + contents: text.as_bytes().to_vec(), + } +} + +/// Polls `request` once with a waker that counts on `woken`. +fn poll( + request: &mut BoxFuture<'_, Result>, + woken: &Arc, +) -> Poll> { + request + .as_mut() + .poll(&mut Context::from_waker(&waker_of(woken))) +} + +/// Whether the run's queue holds a call now. +fn announced(asked: &Asked) -> Option { + match asked.poll_next(&mut Context::from_waker(Waker::noop())) { + Poll::Ready(call) => Some(call), + Poll::Pending => None, + } +} + +#[test] +fn a_request_joins_its_desk_at_its_first_poll_and_resolves_with_the_outcome_the_loop_delivers() { + let asked = Arc::new(Asked::default()); + let call = call_id(); + let desk = CallDesk::open(call, &asked); + let woken = Arc::new(Woken::default()); + + let mut request = desk.request(write("/a.md", "a")); + assert!( + desk.next_request().is_none(), + "an unpolled request is not at the desk" + ); + assert_eq!(announced(&asked), None); + + assert!(poll(&mut request, &woken).is_pending()); + assert_eq!(announced(&asked), Some(call), "the call is announced once"); + assert_eq!(announced(&asked), None); + assert_eq!(desk.next_request(), Some(write("/a.md", "a"))); + assert!(desk.next_request().is_none()); + assert!(poll(&mut request, &woken).is_pending(), "no outcome yet"); + assert_eq!(woken.count(), 0); + + desk.deliver(Ok(VfsOutcome::Unit)); + assert_eq!(woken.count(), 1, "the outcome wakes the request's future"); + assert_eq!( + poll(&mut request, &woken), + Poll::Ready(Ok(VfsOutcome::Unit)) + ); + assert!( + matches!( + poll(&mut request, &woken), + Poll::Ready(Err(VfsError::Backend { .. })) + ), + "a resolved request has no second outcome" + ); +} + +#[test] +fn the_requests_of_one_call_resolve_in_the_order_they_went_out() { + let asked = Arc::new(Asked::default()); + let call = call_id(); + let desk = CallDesk::open(call, &asked); + let woken = Arc::new(Woken::default()); + let mut first = desk.request(write("/first.md", "1")); + let mut second = desk.request(write("/second.md", "2")); + // The second future is polled first, so its request is the older. + assert!(poll(&mut second, &woken).is_pending()); + assert!(poll(&mut first, &woken).is_pending()); + assert_eq!(announced(&asked), Some(call)); + assert_eq!(announced(&asked), Some(call), "one entry for each request"); + + assert_eq!(desk.next_request(), Some(write("/second.md", "2"))); + assert_eq!(desk.next_request(), Some(write("/first.md", "1"))); + let missing = VfsError::NotFound { + path: "/second.md".to_owned(), + }; + desk.deliver(Err(missing.clone())); + assert!(poll(&mut first, &woken).is_pending(), "its outcome is next"); + assert_eq!(poll(&mut second, &woken), Poll::Ready(Err(missing))); + desk.deliver(Ok(VfsOutcome::Bool(true))); + assert_eq!( + poll(&mut first, &woken), + Poll::Ready(Ok(VfsOutcome::Bool(true))) + ); +} + +#[test] +fn a_request_whose_future_is_dropped_before_the_loop_takes_it_is_never_reported() { + let asked = Arc::new(Asked::default()); + let call = call_id(); + let desk = CallDesk::open(call, &asked); + let woken = Arc::new(Woken::default()); + let mut dropped = desk.request(write("/dropped.md", "x")); + let mut kept = desk.request(write("/kept.md", "y")); + assert!(poll(&mut dropped, &woken).is_pending()); + assert!(poll(&mut kept, &woken).is_pending()); + drop(dropped); + + // The run's queue still holds the dropped request's entry, and the + // desk hands the loop the request that is left. + assert_eq!(announced(&asked), Some(call)); + assert_eq!(desk.next_request(), Some(write("/kept.md", "y"))); + assert_eq!(announced(&asked), Some(call)); + assert!(desk.next_request().is_none(), "the dropped request is gone"); + desk.deliver(Ok(VfsOutcome::Unit)); + assert_eq!(poll(&mut kept, &woken), Poll::Ready(Ok(VfsOutcome::Unit))); +} + +#[test] +fn the_outcome_of_a_request_whose_future_is_dropped_after_the_loop_took_it_is_discarded() { + let asked = Arc::new(Asked::default()); + let desk = CallDesk::open(call_id(), &asked); + let woken = Arc::new(Woken::default()); + let mut dropped = desk.request(write("/dropped.md", "x")); + let mut kept = desk.request(write("/kept.md", "y")); + assert!(poll(&mut dropped, &woken).is_pending()); + assert!(poll(&mut kept, &woken).is_pending()); + assert_eq!(desk.next_request(), Some(write("/dropped.md", "x"))); + assert_eq!(desk.next_request(), Some(write("/kept.md", "y"))); + drop(dropped); + + // The operation still happened. Its outcome reaches no one, and the + // next outcome is the kept request's. + desk.deliver(Ok(VfsOutcome::Bool(false))); + assert_eq!(woken.count(), 0, "no future waits on the first outcome"); + assert!(poll(&mut kept, &woken).is_pending()); + desk.deliver(Ok(VfsOutcome::Unit)); + assert_eq!(woken.count(), 1); + assert_eq!(poll(&mut kept, &woken), Poll::Ready(Ok(VfsOutcome::Unit))); + desk.deliver(Ok(VfsOutcome::Unit)); + assert_eq!( + woken.count(), + 1, + "an outcome no request is owed wakes no one" + ); +} + +#[test] +fn a_request_wakes_a_loop_that_waits_on_the_empty_queue() { + let asked = Arc::new(Asked::default()); + let call = call_id(); + let desk = CallDesk::open(call, &asked); + let loop_woken = Arc::new(Woken::default()); + assert!( + asked + .poll_next(&mut Context::from_waker(&waker_of(&loop_woken))) + .is_pending() + ); + + let mut request = desk.request(write("/a.md", "a")); + assert_eq!(loop_woken.count(), 0, "an unpolled request wakes no one"); + assert!(poll(&mut request, &Arc::new(Woken::default())).is_pending()); + assert_eq!(loop_woken.count(), 1); + assert_eq!(announced(&asked), Some(call)); +} diff --git a/crates/harness-internal/runner/src/tool_files.rs b/crates/harness-internal/runner/src/tool_files.rs new file mode 100644 index 000000000..d5ece882c --- /dev/null +++ b/crates/harness-internal/runner/src/tool_files.rs @@ -0,0 +1,284 @@ +//! A tool call's files at the effect loop: the desk through which one +//! call asks for file operations, and the run-wide queue of the calls +//! that asked. +//! +//! Each tool call gets a [`CallDesk`], which is the `ToolFiles` of the +//! call's `ToolContext`. A request joins the desk when its future is +//! first polled, and the call's id joins the run's [`Asked`] queue, which +//! wakes the effect loop. The loop takes the call's oldest request off +//! the desk and reports it to the run, and the run issues it as a `Vfs` +//! effect whose asker is the call. The loop performs that effect inline, +//! records its answer, and hands the whole outcome to the desk, which +//! wakes the request's future. The run issues the effects of one call in +//! the order the loop reported its requests, so the desk matches each +//! outcome to the oldest request it sent. +//! +//! A request whose future is dropped before the loop takes it is never +//! reported. One dropped later stays reported, so the loop still performs +//! it while its call is in flight, and the desk discards its outcome. A +//! desk lives as long as its call is in flight: the loop drops what a +//! call that has landed or was aborted still asks for. +//! +//! The queue also counts the requests the loop takes, and tells the loop +//! when it owes its executor a turn: once in every +//! [`REQUESTS_PER_TURN`] requests. While the recorder answers at once, +//! the loop performs a request and wakes its call without returning to +//! the executor, so calls that keep asking would otherwise hold the +//! executor's thread for as long as they ask. + +use std::collections::{HashMap, VecDeque}; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::task::{Context, Poll, Waker}; + +use runvil_engine::BoxFuture; +use runvil_engine::effect::EffectId; +use runvil_engine::vfs::{VfsError, VfsOp, VfsOutcome}; +use runvil_plugin::ToolFiles; + +/// How many file requests the effect loop takes before it gives its +/// executor a turn. +const REQUESTS_PER_TURN: u32 = 32; + +/// The run-wide queue of the tool calls that asked for a file operation: +/// one entry for each request, oldest first. +#[derive(Default)] +pub(crate) struct Asked { + queue: Mutex, +} + +/// The calls that asked, the effect loop's waker, and the count of the +/// requests the loop took. +#[derive(Default)] +struct AskedQueue { + /// The id of the call behind each request the loop has not taken. + calls: VecDeque, + /// The loop's waker, kept from the last poll that found no call. + waker: Option, + /// The requests whose calls the loop took from the queue since it + /// last gave its executor a turn. + taken: u32, +} + +impl Asked { + /// Adds `call`, which made one more request, and wakes the loop. + fn push(&self, call: EffectId) { + let waker = { + let mut queue = lock(&self.queue); + queue.calls.push_back(call); + queue.waker.take() + }; + if let Some(waker) = waker { + waker.wake(); + } + } + + /// Polls for the call behind the oldest request the loop has not + /// taken. An empty queue keeps `cx`'s waker for the next request. + pub(crate) fn poll_next(&self, cx: &mut Context<'_>) -> Poll { + let mut queue = lock(&self.queue); + if let Some(call) = queue.calls.pop_front() { + return Poll::Ready(call); + } + queue.waker = Some(cx.waker().clone()); + Poll::Pending + } + + /// Counts one request whose call the loop took from the queue, and + /// says whether the loop owes its executor a turn before it reports + /// that request: once in every [`REQUESTS_PER_TURN`] requests of the + /// run. + pub(crate) fn turn_due(&self) -> bool { + let mut queue = lock(&self.queue); + queue.taken += 1; + if queue.taken < REQUESTS_PER_TURN { + return false; + } + queue.taken = 0; + true + } +} + +/// One tool call's desk: the requests the call made, from the first poll +/// of each one's future until that future takes its outcome or is +/// dropped. +pub(crate) struct CallDesk { + /// The id of the call's `ToolCall` effect. + call: EffectId, + /// The run's queue of the calls that asked. + asked: Arc, + requests: Mutex, +} + +/// The requests of one desk. +#[derive(Default)] +struct Requests { + /// The ticket the next request takes. + next: u64, + /// The requests the loop has not taken, oldest first. + queued: VecDeque<(u64, VfsOp)>, + /// The requests the loop reported to the run, oldest first, each + /// owed an outcome. + sent: VecDeque, + /// Each request a future still waits on: its waker until its outcome + /// arrives, then the outcome. + waiting: HashMap, +} + +/// What a request's future waits with. +enum Waiting { + /// The waker of the future's last poll. + For(Waker), + /// The outcome, which the future's next poll takes. + Answered(Result), +} + +impl CallDesk { + /// Opens the desk of tool call `call`, whose requests are announced + /// on `asked`. + pub(crate) fn open(call: EffectId, asked: &Arc) -> Arc { + Arc::new(CallDesk { + call, + asked: Arc::clone(asked), + requests: Mutex::default(), + }) + } + + /// Takes the call's oldest request for the loop to report to the run, + /// and notes that it is owed an outcome. `None` when every request + /// was taken or dropped. + pub(crate) fn next_request(&self) -> Option { + let mut requests = lock(&self.requests); + let (ticket, op) = requests.queued.pop_front()?; + requests.sent.push_back(ticket); + Some(op) + } + + /// Hands `outcome` to the oldest request the loop reported, and wakes + /// the future that waits on it. The outcome of a request whose future + /// was dropped is discarded. + pub(crate) fn deliver(&self, outcome: Result) { + let waker = { + let mut requests = lock(&self.requests); + let Some(ticket) = requests.sent.pop_front() else { + return; + }; + let Some(waiting) = requests.waiting.get_mut(&ticket) else { + return; + }; + match std::mem::replace(waiting, Waiting::Answered(outcome)) { + Waiting::For(waker) => Some(waker), + Waiting::Answered(_) => None, + } + }; + if let Some(waker) = waker { + waker.wake(); + } + } + + /// Queues `op` as a new request, with `waker` to wake at its outcome, + /// announces the call on the run's queue, and returns the request's + /// ticket. + fn ask(&self, op: VfsOp, waker: &Waker) -> u64 { + let ticket = { + let mut requests = lock(&self.requests); + let ticket = requests.next; + requests.next += 1; + requests.queued.push_back((ticket, op)); + requests.waiting.insert(ticket, Waiting::For(waker.clone())); + ticket + }; + self.asked.push(self.call); + ticket + } + + /// Takes the outcome of request `ticket`, or keeps `waker` to wake at + /// its arrival. + fn take(&self, ticket: u64, waker: &Waker) -> Option> { + let mut requests = lock(&self.requests); + match requests.waiting.remove(&ticket)? { + Waiting::Answered(outcome) => Some(outcome), + Waiting::For(_) => { + requests.waiting.insert(ticket, Waiting::For(waker.clone())); + None + } + } + } + + /// Forgets request `ticket`, whose future was dropped: a request the + /// loop has not taken leaves the queue, and the outcome of one it has + /// taken has no one to reach. + fn forget(&self, ticket: u64) { + let mut requests = lock(&self.requests); + requests.queued.retain(|(queued, _)| *queued != ticket); + requests.waiting.remove(&ticket); + } +} + +impl ToolFiles for CallDesk { + fn request(&self, op: VfsOp) -> BoxFuture<'_, Result> { + Box::pin(Request { + desk: self, + stage: Stage::Unsent(op), + }) + } +} + +/// One request's future. +struct Request<'a> { + desk: &'a CallDesk, + stage: Stage, +} + +/// How far one request has come. +enum Stage { + /// Not polled yet: the operation to ask for. + Unsent(VfsOp), + /// At the desk under this ticket. + Asked(u64), + /// Resolved. + Done, +} + +impl Future for Request<'_> { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let request = self.get_mut(); + match std::mem::replace(&mut request.stage, Stage::Done) { + Stage::Unsent(op) => { + request.stage = Stage::Asked(request.desk.ask(op, cx.waker())); + Poll::Pending + } + Stage::Asked(ticket) => { + if let Some(outcome) = request.desk.take(ticket, cx.waker()) { + return Poll::Ready(outcome); + } + request.stage = Stage::Asked(ticket); + Poll::Pending + } + Stage::Done => Poll::Ready(Err(VfsError::Backend { + message: "a file request was polled after it resolved".to_owned(), + })), + } + } +} + +impl Drop for Request<'_> { + fn drop(&mut self) { + if let Stage::Asked(ticket) = self.stage { + self.desk.forget(ticket); + } + } +} + +/// Poison-safe lock: a panic in a tool leaves the queues as they were, +/// and the loop still drains them. +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(PoisonError::into_inner) +} + +#[cfg(test)] +#[path = "tool_files-tests.rs"] +mod tests; diff --git a/crates/harness-internal/runner/tests/it/effect_loop-cap.rs b/crates/harness-internal/runner/tests/it/effect_loop-cap.rs index 4f3fed561..1213a98f3 100644 --- a/crates/harness-internal/runner/tests/it/effect_loop-cap.rs +++ b/crates/harness-internal/runner/tests/it/effect_loop-cap.rs @@ -20,9 +20,9 @@ use runvil_engine::effect::ToolCallOrigin; use runvil_engine::model::{ Completion, CompletionResult, ModelId, ModelInfo, ModelRole, ThinkingMode, ToolCall, }; -use runvil_engine::vfs::Access; use runvil_engine::{BoxFuture, Prompt, Run, RunParams, Timestamp}; use runvil_engine::{ToolCatalog, ToolId, ToolInfo, ToolOutput}; +use runvil_plugin::ToolFiles; use serde_json::{Value, json}; use tokio::sync::{Notify, Semaphore}; @@ -131,9 +131,9 @@ impl ToolPerformer for ParkingTool { &self, _tool: ToolId, _alias: String, - _access: Arc, _origin: ToolCallOrigin, args: Value, + _files: Arc, ) -> BoxFuture<'static, Result> { let name = args["name"].as_str().unwrap_or_default().to_owned(); let gate = Arc::clone(&self.gate); diff --git a/crates/harness-internal/runner/tests/it/effect_loop-vfs.rs b/crates/harness-internal/runner/tests/it/effect_loop-vfs.rs index 7e0627fe1..67386fa4e 100644 --- a/crates/harness-internal/runner/tests/it/effect_loop-vfs.rs +++ b/crates/harness-internal/runner/tests/it/effect_loop-vfs.rs @@ -275,6 +275,54 @@ async fn a_step_with_a_vfs_effect_records_its_inline_answer_before_the_next_effe ); } +/// The operation a store's `Vfs` effect record holds, read from the +/// record's JSON alone: one object keyed by the operation's name, with +/// the arguments of the `store` call as its fields. +fn recorded_op(op: &serde_json::Value) -> VfsOp { + let (name, fields) = op + .as_object() + .and_then(|keyed| keyed.iter().next()) + .expect("an operation records as one object keyed by its name"); + let text = |field: &str| { + fields[field] + .as_str() + .unwrap_or_else(|| panic!("{name} records `{field}` as text: {fields}")) + .to_owned() + }; + let line = |field: &str| fields[field].as_i64(); + match name.as_str() { + "Write" => VfsOp::Write { + path: text("path"), + contents: text("contents").into_bytes(), + }, + "Append" => VfsOp::Append { + path: text("path"), + contents: text("contents").into_bytes(), + }, + "Read" => VfsOp::Read { + path: text("path"), + start: line("start"), + end: line("end"), + }, + "ReadNumbered" => VfsOp::ReadNumbered { + path: text("path"), + start: line("start"), + end: line("end"), + }, + "StrReplace" => VfsOp::StrReplace { + path: text("path"), + old: text("old"), + new: text("new"), + }, + "Delete" => VfsOp::Delete { path: text("path") }, + "Glob" => VfsOp::Glob { + pattern: text("pattern"), + }, + "Exists" => VfsOp::Exists { path: text("path") }, + other => panic!("no `store` call records as {other}"), + } +} + /// A section that writes, appends to, edits, and deletes store files, /// reads the deleted one under a `pcall`, and returns what it reads back /// of the file it kept. @@ -326,9 +374,8 @@ async fn a_runs_vfs_records_rebuild_its_store() { if answer.payload["Vfs"].get("Ok").is_none() { continue; } - let op: VfsOp = serde_json::from_value(vfs["op"].clone()) - .expect("a Vfs effect's record holds its operation whole"); - perform_vfs_op(&files, op).expect("an operation the store performed applies again"); + perform_vfs_op(&files, &recorded_op(&vfs["op"])) + .expect("an operation the store performed applies again"); applied += 1; } assert_eq!(applied, 6, "every operation but the refused read"); diff --git a/crates/harness-internal/runner/tests/it/harness-files-order.rs b/crates/harness-internal/runner/tests/it/harness-files-order.rs new file mode 100644 index 000000000..8d2e6514f --- /dev/null +++ b/crates/harness-internal/runner/tests/it/harness-files-order.rs @@ -0,0 +1,197 @@ +//! The order of one reply's tool calls when a tool changes files: two +//! changes to one file both apply, in the order the model listed them; a +//! call listed before a change is answered before the change starts, and +//! one listed after it starts once the change is answered; and calls +//! that change no file are still in flight together. + +use std::collections::HashMap; +use std::sync::Arc; + +use harness_runner::RunSetup; +use harness_runner::recorder::{MemoryRecorder, Record, RecordKind}; +use runvil_engine::model::{Completion, CompletionResult, ToolCall}; +use runvil_engine::vfs::{Origin, VfsRef}; +use serde_json::{Value, json}; + +use super::files::files_host; +use super::{PendingTimer, launch, setup}; +use crate::scripted::{MODEL, ScriptedBroker, reply}; + +/// A prompt that offers the fixture Pack's reader and its changing tool +/// to one model loop and returns once the loop ends. +const LOOPS: &str = "---\nname: orders\ndescription: d\nrunvil: 0\n\ + uses:\n - tests/files\n---\n\n\ + ## Only\n\n```lua\n\ + tools.offer('files/cat')\n\ + tools.offer('files/append')\n\ + messages:user('go')\n\ + models.loop()\n\ + return 'done'\n```\n"; + +/// The reader's wire name, as the model calls it. +const CAT: &str = "files_cat"; + +/// The changing tool's wire name, as the model calls it. +const APPEND: &str = "files_append"; + +/// A broker whose model asks for `calls` in its first round, each a +/// tool's wire name beside its arguments, and replies with text in every +/// later round. +fn asking(calls: Vec<(&'static str, Value)>) -> ScriptedBroker { + ScriptedBroker::new(move |round, _messages| { + let calls = calls.clone(); + Box::pin(async move { + if round.id.get() > 0 { + return reply("done"); + } + let calls = calls + .into_iter() + .enumerate() + .map(|(index, (name, args))| ToolCall::from_parts(format!("c{index}"), name, args)) + .collect::, _>>()?; + Completion::from_result(CompletionResult::ToolCalls(calls), MODEL).map(Box::new) + }) + }) +} + +/// When each tool call of `records` started and ended, in record order: +/// `start