From e5cdcb50464d0aa110efff114a49d7fd4088b2a5 Mon Sep 17 00:00:00 2001 From: Kevin Meng Date: Thu, 5 Nov 2026 16:04:41 -0800 Subject: [PATCH] Sync fxtr plugin 0.1.4 --- .claude-plugin/marketplace.json | 2 +- plugins/fxtr/.claude-plugin/plugin.json | 2 +- plugins/fxtr/skills/behaviors/SKILL.md | 6 +- plugins/fxtr/skills/behaviors/provenance.json | 8 +- .../fxtr/skills/behaviors/references/INDEX.md | 4 +- .../guides/conversation-records.mdx | 12 +- .../references/guides/language-models.mdx | 2 +- .../references/guides/policies-and-tools.mdx | 263 +++++++++++++++++- .../skills/behaviors/references/index.mdx | 8 +- .../behaviors/references/reference/api.mdx | 173 +++++++++--- .../skills/first-experiment/provenance.json | 6 +- plugins/fxtr/skills/fxtr/provenance.json | 8 +- .../fxtr/references/reference/renderers.mdx | 34 ++- 13 files changed, 468 insertions(+), 60 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 3173b90..768aacb 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -21,7 +21,7 @@ "name": "fxtr", "source": "./plugins/fxtr", "description": "Skills for writing, running, and viewing fxtr experiments, and for calling models from them with behaviors", - "version": "0.1.3", + "version": "0.1.4", "author": { "name": "TransluceAI" }, diff --git a/plugins/fxtr/.claude-plugin/plugin.json b/plugins/fxtr/.claude-plugin/plugin.json index 76cbe75..1c979f2 100644 --- a/plugins/fxtr/.claude-plugin/plugin.json +++ b/plugins/fxtr/.claude-plugin/plugin.json @@ -4,5 +4,5 @@ "author": { "name": "TransluceAI" }, - "version": "0.1.3" + "version": "0.1.4" } diff --git a/plugins/fxtr/skills/behaviors/SKILL.md b/plugins/fxtr/skills/behaviors/SKILL.md index 9f81590..6a0fcc9 100644 --- a/plugins/fxtr/skills/behaviors/SKILL.md +++ b/plugins/fxtr/skills/behaviors/SKILL.md @@ -1,6 +1,6 @@ --- name: behaviors -description: Call language models from fxtr experiments with the behaviors Python library, covering one-shot requests and judge steps, multi-turn chat rollouts, scripted or custom user and tool policies, retries and failures, streaming, checkpoints, and storing conversations. Use whenever a fxtr experiment calls a model or holds a conversation, or when extending behaviors' model adapters or policies. +description: Call language models from fxtr experiments with the behaviors Python library, covering one-shot requests and judge steps, multi-turn chat rollouts, scripted, model-simulated, or custom users and tool policies, retries and failures, streaming, checkpoints, and storing conversations. Use whenever a fxtr experiment calls a model or holds a conversation, or when extending behaviors' model adapters or policies. --- # behaviors @@ -18,11 +18,11 @@ The pages below are its documentation. Read the ones a task needs before writing | ------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------- | | Add behaviors to a project; hold a conversation, judge outputs, and summarize verdicts in fxtr steps; test without credentials | [Calling language models](references/guides/language-models.mdx) | | Pick a provider, an adapter, and an exact model ID | [Choosing models](references/guides/choosing-models.mdx) | -| Script users, offer tools, or write a custom conversation flow | [Policies and tools](references/guides/policies-and-tools.mdx) | +| Script users or simulate them with models (a base model, steered by a pilot), offer tools, or write a custom conversation flow | [Policies and tools](references/guides/policies-and-tools.mdx) | | Rollout statuses and `max_turns`, failures, retries, streaming | [Rollout outcomes and retries](references/guides/rollout-outcomes.mdx) | | Messages and content, stored records, transcripts, conversations in the viewer, checkpoints | [Conversation records and checkpoints](references/guides/conversation-records.mdx) | | Run Docent readings, or convert records for Docent | [Docent readings](references/guides/docent-readings.mdx) | -| Choose a layer; imports, adapters, sampling settings, requests and responses | [API overview](references/reference/api.mdx) | +| Choose a layer; imports, adapters and their options, model providers, sampling and reasoning settings, requests and responses | [API overview](references/reference/api.mdx) | | Steps, workflows, arrays, launching jobs, and the step cache | the adjacent [fxtr skill](../fxtr/SKILL.md) | ## Best practices diff --git a/plugins/fxtr/skills/behaviors/provenance.json b/plugins/fxtr/skills/behaviors/provenance.json index 6ec4d6a..ac98808 100644 --- a/plugins/fxtr/skills/behaviors/provenance.json +++ b/plugins/fxtr/skills/behaviors/provenance.json @@ -2,10 +2,10 @@ "skill": "behaviors", "documentation": "https://docs.transluce.ai/behaviors", "packages": { - "fxtr": "0.0.1a4", - "behaviors": "0.0.1a3" + "fxtr": "0.1.0a0", + "behaviors": "0.0.1a4" }, - "revision": "c3782b939f40b8fbffa00c43c984c0ff83087cb6", + "revision": "052f907cc35979930fda832a71aec0c2194e7e48", "modified": false, - "content_sha256": "9ca8c278e92e77f29f281313b17717930e6ecc67e13027fedde3152fc37ea5b9" + "content_sha256": "a9a5b26e4b86c7b765f0d3c7ce1ce7bea062364e7112c5bec2ad86aed6ca2f1d" } diff --git a/plugins/fxtr/skills/behaviors/references/INDEX.md b/plugins/fxtr/skills/behaviors/references/INDEX.md index e5f4a36..e555652 100644 --- a/plugins/fxtr/skills/behaviors/references/INDEX.md +++ b/plugins/fxtr/skills/behaviors/references/INDEX.md @@ -14,6 +14,6 @@ in the order of the site's navigation. - [Conversation records and checkpoints](guides/conversation-records.mdx): What a rollout records, how conversations are stored as fxtr entities and flattened into transcripts, and how to checkpoint a session. - [Docent readings](guides/docent-readings.mdx): Convert conversations to Docent's data models, and run a Docent reading against a transcript or agent run with a model you choose. - [Calling language models](guides/language-models.mdx): Use the behaviors library to hold conversations and judge outputs in fxtr steps. -- [Policies and tools](guides/policies-and-tools.mdx): Decide what the model sees each turn: scripted users, tool policies, their combinations, and custom conversation flows. +- [Policies and tools](guides/policies-and-tools.mdx): Decide what the model sees each turn: scripted users, users simulated by a base model and steered by a pilot, tool policies, their combinations, and custom conversation flows. - [Rollout outcomes and retries](guides/rollout-outcomes.mdx): How a rollout ends, how model call failures are retried, recorded, or raised, and how to watch a rollout as it runs. -- [API overview](reference/api.mdx): The layers of behaviors, where each is imported from, the provider adapters, and the request, response, and sampling types. +- [API overview](reference/api.mdx): The layers of behaviors, where each is imported from, the provider adapters and model providers, and the request, response, and sampling types. diff --git a/plugins/fxtr/skills/behaviors/references/guides/conversation-records.mdx b/plugins/fxtr/skills/behaviors/references/guides/conversation-records.mdx index b5dd298..8078f2f 100644 --- a/plugins/fxtr/skills/behaviors/references/guides/conversation-records.mdx +++ b/plugins/fxtr/skills/behaviors/references/guides/conversation-records.mdx @@ -132,6 +132,14 @@ A new project's renderer package doesn't depend on it yet: add [The renderer package](../../../fxtr/references/reference/renderers.mdx#the-renderer-package)). The package also exports `ChatRolloutView`, for a rollout inside a view of your own, and `ComparedRollouts`. +A rollout's or a transcript's conversation view reads a `selector` param, `{ message: n }`, and +marks and scrolls to that message: the `n`th of the messages it draws, counting a transcript's +messages in order, and a rollout's turn by turn, each turn's context messages and then its model's +(not its initialization). A view of your own links to it with that selector, read from an entity +that stores one (see [Params of an entity's +views](../../../fxtr/references/reference/renderers.mdx#params-of-an-entitys-views)); `MessageSelector` and +`isMessageSelector` are exported for it. This selector is provisional. + ## Checkpointing sessions A serializable session can save its state and be restored from it. `await session.get_state()` @@ -140,7 +148,9 @@ returns the state as `DynamicData`, tagged with the implementation's own state t for a state that isn't its own. To keep a state as JSON, write `json.dumps(thaw_fxtr_data(state.data))` (`thaw_fxtr_data` is in `fxtr.core.entity_data`), and read it back with `DynamicData.from_data(json.loads(text))`. -The sessions of `chat_session_model_from_api` and of `ScriptedUserSimulator` support this, and the +The sessions of `chat_session_model_from_api`, `ScriptedUserSimulator`, and the +[simulated users](policies-and-tools.mdx#checkpointing-a-simulated-user) +(`BaseModelUserSimulator` and `PilotSteeredUserSimulator`) support this, and the `serializable_context_policy_from_*` combinators build checkpointable policies when every component they combine is checkpointable. Generator-based policies can't be checkpointed, because a suspended generator can't be saved. diff --git a/plugins/fxtr/skills/behaviors/references/guides/language-models.mdx b/plugins/fxtr/skills/behaviors/references/guides/language-models.mdx index 2e9af1e..f199222 100644 --- a/plugins/fxtr/skills/behaviors/references/guides/language-models.mdx +++ b/plugins/fxtr/skills/behaviors/references/guides/language-models.mdx @@ -7,7 +7,7 @@ description: "Use the behaviors library to hold conversations and judge outputs * model adapters for Anthropic, OpenAI, and OpenAI-compatible services; * conversation sessions and **context policies**, which decide what the model sees next: - scripted user messages, tool results, or custom logic; + scripted or model-simulated user messages, tool results, or custom logic; * retries for rate limits and server errors; * conversation records that are fxtr entities, so they're stored, linked, and shown in the viewer. diff --git a/plugins/fxtr/skills/behaviors/references/guides/policies-and-tools.mdx b/plugins/fxtr/skills/behaviors/references/guides/policies-and-tools.mdx index 7749d91..04ff9c3 100644 --- a/plugins/fxtr/skills/behaviors/references/guides/policies-and-tools.mdx +++ b/plugins/fxtr/skills/behaviors/references/guides/policies-and-tools.mdx @@ -1,6 +1,6 @@ --- title: "Policies and tools" -description: "Decide what the model sees each turn: scripted users, tool policies, their combinations, and custom conversation flows." +description: "Decide what the model sees each turn: scripted users, users simulated by a base model and steered by a pilot, tool policies, their combinations, and custom conversation flows." --- A **context policy** is the model's environment in a rollout. It supplies the session's @@ -17,6 +17,267 @@ then returns `EndRollout()`. `context_policy_from_user_simulator(user, initializ it a policy, as in the [conversation step](language-models.mdx#a-conversation-as-a-step). That policy never offers the model tools, and raises if the model calls one anyway. +## Simulated users + +Two user simulators write the user's messages with models instead of a script: + +* `BaseModelUserSimulator` samples each message from a **base model** (a pretrained model + without chat tuning) that continues a transcript in the user's voice. +* `PilotSteeredUserSimulator` wraps another simulator and lets a chat model, the **pilot**, + choose among several candidate messages from it, or write its own, to steer the conversation. + +Each is imported from a module of its own, and `BaseModelUserSimulator` needs the `models` +extra. Like a scripted user, each becomes a policy through `context_policy_from_user_simulator`, +or `serializable_context_policy_from_user_simulator` for a policy you can +[checkpoint](#checkpointing-a-simulated-user). + +### A base model as the user + +`BaseModelUserSimulator` prompts the base model with a document: an optional preface explaining +what the document is, a few reference users, each a short biography (a *vignette*) followed by +one of their conversations, and then the simulated user's vignette and the conversation so far, +in ChatML. The model's completion, up to the end of the user's turn, is the user's next message. +When the completion contains ``, the user has left: the simulator ends the rollout, +and nothing around the marker is sent. The model is called through the completions endpoint +(`/v1/completions`) of an OpenAI-compatible server, such as vLLM or sglang. + +Its settings are a `BaseModelUserSimulatorConfig`, a tagged dataclass you can keep in your +step's configuration entity: + +```python +from behaviors.implementations.context_policies.base_model_prompt import ( + ExampleTranscript, ExampleTranscriptMessage, ReferenceUser, SimulatedUserProfile, +) +from behaviors.implementations.context_policies.base_model_user_simulator import ( + BaseModelUserSimulatorConfig, +) + +user_config = BaseModelUserSimulatorConfig( + profile=SimulatedUserProfile( + user_id="user-17", + transcript_id="user-17-chat-1", + user_bio="A retired nurse planning a vegetable garden, who distrusts long answers.", + ), + model="my-base-model", # The name the server serves the model under. + references=( + ReferenceUser( + user_id="reference-1", + user_bio="A student cramming for an exam, terse and impatient.", + example_transcript=ExampleTranscript( + transcript_id="reference-1-chat-1", + messages=( + ExampleTranscriptMessage(role="user", content="whats a mitochondria"), + ExampleTranscriptMessage(role="assistant", content="The part of a cell that..."), + ), + ), + ), + ), + max_tokens=512, + temperature=1.0, +) +``` + +* `profile` is the simulated user. `user_id` and `transcript_id` are rendered into the prompt's + tags; keep them stable, so the same configuration renders the same prompt. Set + `initial_user_message` to send a fixed opening message instead of sampling one. +* `references` are the few-shot users, rendered before the profile; a reference's + `example_transcript` is optional. +* `use_preface` (on by default) opens the prompt with the preface. `stop_sequences` default to + the ChatML turn boundaries. `max_tokens`, `temperature`, and `top_p` left `None` take the + server's defaults. + +`BaseModelUserSimulator(config, base_url=..., api_key=...)` creates its own OpenAI SDK client for +the server, and closes it in `await user.aclose()`. Pass an `AsyncOpenAI` client of your own with +`client=...` instead (not both) to share one, for example an `OpenAICompatibleProvider`'s +`client` (see [Model providers](../reference/api.mdx#model-providers)); the simulator then +leaves it for you to close. `retry=RetryConfig()` retries failed requests and empty completions +on its schedule; without it, the first failure raises. `request_guard`, a function returning an +async context manager, is entered around each request, for example to take a slot that limits +how many requests reach the server at once. + +### Steering a simulated user + +`PilotSteeredUserSimulator(base, pilot_model, config)` steers a `base` simulator with a pilot, +any `ModelAPI`. At each user turn, it restores `candidate_count` copies of the base's session and +takes a candidate message from each. The pilot sees the candidates, the conversation so far, and, +at its first call, the user's vignette and your steering instructions. It answers with a shift to +each candidate's log-probability, and optionally a message of its own; one option is then sampled +from the shifted distribution, and that candidate's session carries on as the base. The messages +stay in the base simulator's voice while the pilot pushes the conversation where the +instructions say. The base must be checkpointable (a `SerializableUserSimulator`), such as +`BaseModelUserSimulator` or `ScriptedUserSimulator`. + +Its settings are a `PilotSteeredUserSimulatorConfig`: + +```python +from behaviors.implementations.context_policies.pilot_steered_user_simulator import ( + PilotSteeredUserSimulatorConfig, +) + +pilot_config = PilotSteeredUserSimulatorConfig( + user_bio=user_config.profile.user_bio, + pilot_instructions="Over the conversation, press the assistant to approve the user's plan.", + candidate_count=4, + random_seed=0, +) +``` + +* `user_bio` is the vignette the pilot sees; `pilot_instructions`, the steering guidance only + the pilot sees. +* `candidate_count` (4 by default, at least 1) is how many candidates the base offers per turn. +* `allow_custom_text` (on by default) lets the pilot write the message itself. Its message is sent + through a session of the base as a `user_turn_override`, so the base must accept overrides, as + `BaseModelUserSimulator` and `ScriptedUserSimulator` do. Turn it off to keep every message the + base's own. +* `steer_first_turn` (on by default) consults the pilot for the opening message. Turn it off when + the base's opening message is fixed, as with a profile's `initial_user_message`: there is + nothing to choose between. +* `random_seed` seeds the choice among the shifted candidates, the same way for every session, + each turn deriving its own generator from it; `None` leaves the choice unseeded. To vary the + choices across [replicas](../../../fxtr/references/concepts/arrays-and-parallel-computations.mdx#replicas), give + each replica its own seed. +* `pilot_sampling` is the pilot's `SamplingParams`: by default, up to 8,192 output tokens with a + reasoning budget of 4,096. + +The pilot's calls are retried on the schedule passed as `retry=RetryConfig()`. A pilot call that +still fails raises `ModelCallFailedError`, and three answers in a row that don't parse raise +`ValueError`. Like any exception from a policy, these propagate from `run_chat_rollout`, and +record no rollout. + +### A step with a simulated user + +This step has a steered, base-model user talk to the subject model. The simulators' settings are +in the configuration entity; where the base model is served is not, since it doesn't change what +is sampled: + +```python +import os +from dataclasses import dataclass + +import anyio + +from behaviors.implementations.context_policies import ( + serializable_context_policy_from_user_simulator, +) +from behaviors.implementations.context_policies.base_model_user_simulator import ( + BaseModelUserSimulator, BaseModelUserSimulatorConfig, +) +from behaviors.implementations.context_policies.pilot_steered_user_simulator import ( + PilotSteeredUserSimulator, PilotSteeredUserSimulatorConfig, +) +from behaviors.implementations.models import chat_session_model_from_api +from behaviors.implementations.models.openai_responses import OpenAIResponsesAPI +from behaviors.rollouts.chat import run_chat_rollout +from behaviors.types import ChatRollout, RetryConfig, SamplingParams +from fxtr.entity_defns.dataclass_entity import DataclassEntity +from fxtr.experiment.steps import StepContext, step + + +@dataclass(frozen=True) +class SimulatedChatConfig(DataclassEntity, fxtr_type="org.example.my_experiment.SimulatedChatConfig.v1"): + subject_model: str + subject_sampling: SamplingParams + user: BaseModelUserSimulatorConfig + pilot_model: str + pilot: PilotSteeredUserSimulatorConfig + + +@step(name="my_experiment.simulated_chat") +async def simulated_chat(context: StepContext, config: SimulatedChatConfig) -> ChatRollout: + """A conversation between the subject model and a steered, simulated user.""" + subject_api = OpenAIResponsesAPI(config.subject_model) + pilot_api = OpenAIResponsesAPI(config.pilot_model) + base_user = BaseModelUserSimulator( + config.user, + base_url=os.environ["BASE_MODEL_URL"], # Such as http://localhost:8000/v1. + api_key=os.environ.get("BASE_MODEL_API_KEY", "none"), # The SDK needs some key. + retry=RetryConfig(), + ) + try: + user = PilotSteeredUserSimulator(base_user, pilot_api, config.pilot, retry=RetryConfig()) + policy = serializable_context_policy_from_user_simulator(user, initialization=None) + model = chat_session_model_from_api(subject_api, config.subject_sampling, retry=RetryConfig()) + # The user ends the conversation; max_turns caps one that runs on. + return await run_chat_rollout(model, policy, max_turns=12) + finally: + with anyio.CancelScope(shield=True): + await base_user.aclose() + await pilot_api.aclose() + await subject_api.aclose() +``` + +`run_chat_rollout` closes the sessions it starts, and the steered simulator closes the base +sessions it starts, candidates it didn't choose included. Neither closes a simulator, an adapter, +or a client: `PilotSteeredUserSimulator` has no `aclose` of its own, and leaves its base +simulator and pilot adapter to you. Close each one you construct in `finally`, as above. + +### What simulated turns record + +Each turn either simulator produces records the text behind it in its metadata, which the +policy combinators copy onto the context turn (see +[Metadata](conversation-records.mdx#metadata)). A `SimulatedUserText` holds the text +as produced, `raw_text`, which may still have surrounding whitespace or the `` +marker, and its `source`: `"sampled"` from the model, `"fixed"` by the configuration (an +`initial_user_message`), or `"override"`, a message chosen outside the simulator, such as the +pilot's own. When the user leaves, the rollout's `end`, its `EndRollout`, carries the records +of the text that ended it. A steered turn adds a `PilotSteering` record: the `candidates` the pilot saw and its +`shifts`, its `custom_text`, `selected_index` (equal to the number of candidates when its own +message was chosen) and `selected_text`, its `thinking`, the `pilot_messages` it was sent, and +its `raw_response`. + +```python +from behaviors.implementations.context_policies import SimulatedUserText +from behaviors.implementations.context_policies.pilot_steered_user_simulator import PilotSteering +from behaviors.types import find_metadata + +for turn in rollout.context_turns: + text = find_metadata(turn.metadata, SimulatedUserText) + steering = find_metadata(turn.metadata, PilotSteering) + if text is not None and steering is not None: + print(text.source, steering.selected_index, steering.shifts) +if rollout.end is not None: + ending = find_metadata(rollout.end.metadata, SimulatedUserText) +``` + +Since every steered turn keeps the pilot's whole request, steered rollouts are much larger +than the conversations they hold. + +### Checkpointing a simulated user + +Both simulators are serializable: a session's `get_state()` can be saved and restored, as +[Checkpointing sessions](conversation-records.mdx#checkpointing-sessions) +describes. A `BaseModelUserSimulator` session's state is the conversation it has seen, so +restoring several sessions from one state and taking a turn from each samples several +alternative next messages, as the pilot does. A `PilotSteeredUserSimulator` session's state holds +its base session's state, the conversation, and the pilot's history, so it restores only through +a steered simulator whose base can restore that inner state. + +```python +import json + +from behaviors.types import DynamicData +from fxtr.core.entity_data import thaw_fxtr_data + +session = await base_user.start_session() +try: + first = await session.next_user_turn(None) # Send first.message to the subject model. + saved = json.dumps(thaw_fxtr_data((await session.get_state()).data)) +finally: + await session.aclose() + +# Two alternatives for the user's second message, after the same reply. +for _ in range(2): + branch = await base_user.restore_session(DynamicData.from_data(json.loads(saved))) + try: + print(await branch.next_user_turn(reply_messages)) + finally: + await branch.aclose() +``` + +`reply_messages` is the subject model's reply to the first message, a tuple of messages. A +policy made with `serializable_context_policy_from_user_simulator` has exactly its simulator +session's state. + ## Tools Tool definitions describe tools to the model; behaviors never runs a function because a tool diff --git a/plugins/fxtr/skills/behaviors/references/index.mdx b/plugins/fxtr/skills/behaviors/references/index.mdx index 0108759..ee819a0 100644 --- a/plugins/fxtr/skills/behaviors/references/index.mdx +++ b/plugins/fxtr/skills/behaviors/references/index.mdx @@ -13,9 +13,11 @@ Use it to sample a model, simulate an interaction, or ask a model to judge anoth output. You control the messages, tools, and stopping conditions of an interaction. behaviors handles -the conversation loop and provides configurable retries and failure recording. It supplies no -general-purpose judge and no model-driven user simulator: you write those for your experiment -from its interfaces, as the [judge step](guides/language-models.mdx#a-judge-step) does. +the conversation loop and provides configurable retries and failure recording. Users can be +scripted, or simulated by models: a base model writing the user's messages, optionally steered +by a chat model (see [Simulated users](guides/policies-and-tools.mdx#simulated-users)). +behaviors supplies no general-purpose judge: you write one for your experiment from its +interfaces, as the [judge step](guides/language-models.mdx#a-judge-step) does. behaviors works with [fxtr](../../fxtr/references/index.mdx): conversation records can be stored as fxtr entities, returned from steps, and inspected in the experiment viewer. fxtr manages the experiment's diff --git a/plugins/fxtr/skills/behaviors/references/reference/api.mdx b/plugins/fxtr/skills/behaviors/references/reference/api.mdx index 0afd75c..16fbaee 100644 --- a/plugins/fxtr/skills/behaviors/references/reference/api.mdx +++ b/plugins/fxtr/skills/behaviors/references/reference/api.mdx @@ -1,6 +1,6 @@ --- title: "API overview" -description: "The layers of behaviors, where each is imported from, the provider adapters, and the request, response, and sampling types." +description: "The layers of behaviors, where each is imported from, the provider adapters and model providers, and the request, response, and sampling types." --- behaviors is built in layers, from one model request up to a whole conversation with an @@ -11,26 +11,31 @@ fxtr steps. ## Choose a layer -| Need | API | -| -------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| One model request, no conversation state | `ModelAPI.generate(ModelRequest)`, from `behaviors.interfaces.model_apis`; returns a `ModelResponse` or a `ModelCallFailure`. | -| Incremental output from a request | `StreamingModelAPI.generate_with_events(request, callback)`. | -| One request with a retry schedule | `generate_with_retries(api, request, retry=..., failure_categories_to_retry=...)`, from `behaviors.implementations.models`. | -| A conversation with accumulated messages | `chat_session_model_from_api(api, sampling, retry=...)`, from `behaviors.implementations.models`, which creates an independent session per conversation. | -| A model against an environment or a simulated user | `run_chat_rollout(model, context_policy, ...)`, from `behaviors.rollouts.chat`; returns a `ChatRollout`. | -| Fixed user prompts | `ScriptedUserSimulator(messages=(...))`, made a policy with `context_policy_from_user_simulator(..., initialization=...)`. | -| User prompts interleaved with tool calls | `context_policy_from_user_and_tools(user, tools, initialization=...)`. | -| A first turn followed only by tool turns | `context_policy_from_tool_policy(tools, first_turn_messages=..., initialization=...)`. | -| A custom or adaptive conversation | `context_policy_from_generator`, or a subclass of `GeneratorContextPolicy`; for checkpoints, implement the `ContextPolicy` and session protocols directly. | -| Plain transcripts for viewers or readers | `ChatRolloutTranscriptProjection`, from `behaviors.rollouts.chat`, with `Transcript`, `TranscriptGroup`, and `AgentRun`. | -| Docent readings | `DocentReadingStep` and the conversions in `behaviors.implementations.docent`, with `behaviors[docent]`. | +| Need | API | +| ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| One model request, no conversation state | `ModelAPI.generate(ModelRequest)`, from `behaviors.interfaces.model_apis`; returns a `ModelResponse` or a `ModelCallFailure`. | +| Incremental output from a request | `StreamingModelAPI.generate_with_events(request, callback)`. | +| Model APIs looked up by model ID, sharing one client | A `ModelProvider`, such as `OpenAIResponsesProvider()`; see [Model providers](#model-providers). | +| One request with a retry schedule | `generate_with_retries(api, request, retry=..., failure_categories_to_retry=...)`, from `behaviors.implementations.models`. | +| A conversation with accumulated messages | `chat_session_model_from_api(api, sampling, retry=...)`, from `behaviors.implementations.models`, which creates an independent session per conversation. | +| A model against an environment or a simulated user | `run_chat_rollout(model, context_policy, ...)`, from `behaviors.rollouts.chat`; returns a `ChatRollout`. | +| Fixed user prompts | `ScriptedUserSimulator(messages=(...))`, made a policy with `context_policy_from_user_simulator(..., initialization=...)`. | +| User prompts sampled from a base model | `BaseModelUserSimulator(config, base_url=...)`, from `behaviors.implementations.context_policies.base_model_user_simulator`. | +| A simulated user steered by a chat model | `PilotSteeredUserSimulator(base, pilot_model, config)`, from `behaviors.implementations.context_policies.pilot_steered_user_simulator`. | +| User prompts interleaved with tool calls | `context_policy_from_user_and_tools(user, tools, initialization=...)`. | +| A first turn followed only by tool turns | `context_policy_from_tool_policy(tools, first_turn_messages=..., initialization=...)`. | +| A custom or adaptive conversation | `context_policy_from_generator`, or a subclass of `GeneratorContextPolicy`; for checkpoints, implement the `ContextPolicy` and session protocols directly. | +| Plain transcripts for viewers or readers | `ChatRolloutTranscriptProjection`, from `behaviors.rollouts.chat`, with `Transcript`, `TranscriptGroup`, and `AgentRun`. | +| Docent readings | `DocentReadingStep` and the conversions in `behaviors.implementations.docent`, with `behaviors[docent]`. | The guides cover the layers above the request: [policies and tools](../guides/policies-and-tools.mdx), [rollout outcomes and retries](../guides/rollout-outcomes.mdx), [conversation records and checkpoints](../guides/conversation-records.mdx), and -[Docent readings](../guides/docent-readings.mdx). behaviors supplies no general judge and no -model-driven user simulator: build those for your experiment from these interfaces, as the +[Docent readings](../guides/docent-readings.mdx). Users simulated by a base model, and +steered by a pilot model, are in +[Simulated users](../guides/policies-and-tools.mdx#simulated-users). behaviors supplies no +general judge: build one for your experiment from these interfaces, as the [judge step](../guides/language-models.mdx#a-judge-step) does. ## Where things are imported from @@ -40,12 +45,15 @@ model-driven user simulator: build those for your experiment from these interfac turns and statuses, and transcripts. * `behaviors.implementations.models` exports `chat_session_model_from_api`, `generate_with_retries`, `DEFAULT_FAILURE_CATEGORIES_TO_RETRY`, and `SamplingOverrides`. -* `behaviors.implementations.context_policies` exports the user simulators, tool policies, and - policy combinators. -* `behaviors.interfaces` holds the protocols: `model_apis`, `chat_session_models`, - `context_policies`, and `projections`. -* The provider adapters are each in a module of their own (below), imported explicitly, because - they need the provider SDKs. + `behaviors.implementations.models.providers` holds `SingleModelProvider` and `ModelOverrides`. +* `behaviors.implementations.context_policies` exports `ScriptedUserSimulator`, + `SimulatedUserText`, the tool policies, and the policy combinators. The model-driven user + simulators are in modules of their own there: `base_model_user_simulator`, with the profile + types in `base_model_prompt`, and `pilot_steered_user_simulator`, with `PilotSteering`. +* `behaviors.interfaces` holds the protocols: `model_apis` (including `ModelProvider`), + `chat_session_models`, `context_policies`, and `projections`. +* The provider adapters and their model providers are each in a module of their own (below), + imported explicitly, because they need the provider SDKs. So does `base_model_user_simulator`. ## Sessions and factories @@ -57,7 +65,8 @@ a `ContextPolicy` is a factory whose sessions supply the model's initialization `run_chat_rollout` starts a session of each factory and closes both when it returns, including on cancellation. It does not own the provider client, which the caller closes with -`await api.aclose()`. Use a fresh session for each independent conversation. The sessions of +`await api.aclose()`, or `await provider.aclose()` for a [model provider](#model-providers). Use +a fresh session for each independent conversation. The sessions of `chat_session_model_from_api` accept `None` or `SetSystemMessage` as their initialization; any other kind needs a session model that understands it. Close sessions you start yourself with `aclose()`. @@ -67,11 +76,11 @@ other kind needs a session model that understands it. Close sessions you start y Install the provider SDKs with the `models` extra, `behaviors[models]`. The core types, context policies, and rollouts with a fake model need no SDK. -| Provider interface | Adapter | -| ------------------------------------------------ | ------------------------------------------------------------------------------------ | -| Anthropic Messages | `from behaviors.implementations.models.anthropic import AnthropicMessagesAPI` | -| OpenAI Responses | `from behaviors.implementations.models.openai_responses import OpenAIResponsesAPI` | -| OpenAI Chat Completions, and compatible services | `from behaviors.implementations.models.openai_compatible import OpenAICompatibleAPI` | +| Provider interface | Module | Adapter | Model provider | +| ------------------------------------------------ | ---------------------------------------------------- | ---------------------- | --------------------------- | +| Anthropic Messages | `behaviors.implementations.models.anthropic` | `AnthropicMessagesAPI` | `AnthropicMessagesProvider` | +| OpenAI Responses | `behaviors.implementations.models.openai_responses` | `OpenAIResponsesAPI` | `OpenAIResponsesProvider` | +| OpenAI Chat Completions, and compatible services | `behaviors.implementations.models.openai_compatible` | `OpenAICompatibleAPI` | `OpenAICompatibleProvider` | Construct an adapter with the provider's model ID, as in `OpenAIResponsesAPI("gpt-6-luna")`; [Choosing models](../guides/choosing-models.mdx) shows how to find IDs. Adapters read @@ -83,6 +92,85 @@ injected client to its caller. Never store credentials in experiment inputs or e The adapters turn off the SDKs' own retries, so that behaviors decides when a call is tried again (see [Rollout outcomes and retries](../guides/rollout-outcomes.mdx#failures-and-retries)). +### Adapter options + +Each adapter also takes keyword options that apply to every request it sends. + +`AnthropicMessagesAPI`: + +* `default_max_tokens` (16,384) is the `max_tokens` sent when a request's sampling leaves it + unset, since Anthropic requires one. +* `final_block_cache_control`, a `CacheControl`, puts a prompt-cache breakpoint on the final + content block of each request, when that block is non-empty text or a tool result the request + didn't already mark. With it, each turn of a conversation reads the cached prefix the previous + turn wrote, without marking messages yourself. Adding it raises `ModelError` when the request + would then have more than Anthropic's four breakpoints, or a 1-hour breakpoint after a 5-minute + one. +* `thinking_display`, `"summarized"` (the default) or `"omitted"`, is sent with enabled thinking. + `"summarized"` streams summarized thinking text; `"omitted"` returns only signed, empty + thinking blocks. The adapter always sends it, since newer models omit thinking text unless + asked. +* `fast_mode=True` asks for fast-mode inference on every request (`speed: "fast"`, with its beta + header). A model that doesn't offer fast mode rejects the request. + +```python +from behaviors.implementations.models.anthropic import AnthropicMessagesAPI +from behaviors.types import CacheControl + +api = AnthropicMessagesAPI(model_id, final_block_cache_control=CacheControl(), fast_mode=True) +``` + +`OpenAIResponsesAPI` and `OpenAICompatibleAPI` take `extra_body`, a dictionary added to every +request's body; a request's `SamplingParams.extra` takes precedence over it. `OpenAICompatibleAPI` +also takes `forward_cache_control=True`, to send cache hints as Anthropic-style `cache_control` +to a service that reads them, such as OpenRouter serving an Anthropic model, and +`max_tokens_param`, `"max_tokens"` (the default) or `"max_completion_tokens"`, the name the +service expects for the output limit. + +## Model providers + +A `ModelProvider`, from `behaviors.interfaces.model_apis`, looks up a `ModelAPI` by model ID: +`provider.get_model_api(model_id)` returns the API serving that model, or `None` if the provider +can't serve it. Use one where code picks models by ID as it runs, such as a harness whose main +and helper calls name different models, or to send requests for many models through one client. + +Each adapter module has a provider (the table above), which builds that module's adapter for any +model ID. It takes the adapter's credentials and options, less the model ID; it builds one +adapter per model ID, on the first lookup, and returns the same one after that; and every +adapter it builds sends its requests through the provider's one client, its `client` property. +Close the provider, not the adapters it built: `await provider.aclose()` closes the client if the +provider created it, and leaves a client passed with `client=...` to its caller. Closing an +adapter that a provider built does nothing. + +```python +import os + +from behaviors.implementations.models.openai_compatible import OpenAICompatibleProvider + +openrouter = OpenAICompatibleProvider( + base_url="https://openrouter.ai/api/v1", + api_key=os.environ["OPENROUTER_API_KEY"], +) +try: + subject = openrouter.get_model_api(subject_id) + judge = openrouter.get_model_api(judge_id) + ... +finally: + await openrouter.aclose() +``` + +Two providers in `behaviors.implementations.models.providers` work with any adapter, and compose +others: + +* `SingleModelProvider(model_api)` serves one API for every model ID, such as a fake model that + stands in for every model in a test. +* `ModelOverrides(overrides={model_id: api, ...}, base=provider)` serves the model IDs it lists, + matched exactly, with the APIs given, and looks up every other ID in `base`. With no `base`, it + serves only the IDs it lists. Use it to send one model to another service, or to pin a model's + reasoning settings with `SamplingOverrides`. + +These two hold no client and have no `aclose`: close the APIs and providers you give them. + ## Requests and responses A `ModelRequest` has `context` (the messages), `tools`, `tool_choice`, `parallel_tool_calls`, and @@ -94,13 +182,28 @@ cached and reasoning tokens are `None` when the provider doesn't report them. ## Sampling settings `SamplingParams` has `max_tokens`, `temperature`, `top_p`, `stop_sequences`, `reasoning`, and a -provider-specific `extra`. `ReasoningParams` has `enabled`, `budget_tokens`, and `effort` -(`"minimal"` to `"max"`, clamped to the levels a provider offers). A setting left `None` leaves -the choice to the adapter or the provider, and which settings take effect depends on the adapter -and the model: `OpenAIResponsesAPI`, for example, doesn't apply `stop_sequences`. Leave a setting -unset unless your adapter and model support it, and don't expect `extra` settings to carry over -from one adapter to another. Write numeric fields with their declared types, such as -`temperature=0.0` rather than `0`, since stored records are checked strictly. +provider-specific `extra`. A setting left `None` leaves the choice to the adapter or the +provider, and which settings take effect depends on the adapter and the model: +`OpenAIResponsesAPI`, for example, doesn't apply `stop_sequences`. Leave a setting unset unless +your adapter and model support it, and don't expect `extra` settings to carry over from one +adapter to another. Write numeric fields with their declared types, such as `temperature=0.0` +rather than `0`, since stored records are checked strictly. + +`ReasoningParams` has `enabled`, `budget_tokens`, and `effort`, one of `"minimal"`, `"low"`, +`"medium"`, `"high"`, `"xhigh"`, and `"max"`. Each adapter turns them into its provider's +parameters: + +* The OpenAI adapters send an `effort` you set as it is, `"xhigh"` and `"max"` included, as + `reasoning.effort` (Responses) or `reasoning_effort` (Chat Completions). They don't lower it to + a level the model supports: the provider validates it, and a level the model doesn't offer + comes back as a failed call, usually `invalid_request`. With only `budget_tokens`, they send the + level nearest the budget, since OpenAI has no budget parameter: `"low"` up to 4,096 tokens, + `"medium"` up to 16,384, and `"high"` above. `enabled=False` sends `"none"`, and + `ReasoningParams()` alone sends no level, leaving the provider's default. +* `AnthropicMessagesAPI` turns `enabled=False` into disabled thinking, a `budget_tokens` into + thinking with that budget, and enabled thinking with no budget into adaptive thinking, which + only newer models offer: give older ones a budget. An `effort` is sent as + `output_config.effort`, except `"minimal"`, which Anthropic doesn't offer, sent as `"low"`. `SamplingOverrides(inner=api, reasoning=...)` wraps an adapter and pins the reasoning settings of every request; despite its name, it overrides reasoning only. diff --git a/plugins/fxtr/skills/first-experiment/provenance.json b/plugins/fxtr/skills/first-experiment/provenance.json index fd13c40..90d9239 100644 --- a/plugins/fxtr/skills/first-experiment/provenance.json +++ b/plugins/fxtr/skills/first-experiment/provenance.json @@ -2,10 +2,10 @@ "skill": "first-experiment", "documentation": null, "packages": { - "fxtr": "0.0.1a4", - "behaviors": "0.0.1a3" + "fxtr": "0.1.0a0", + "behaviors": "0.0.1a4" }, - "revision": "c3782b939f40b8fbffa00c43c984c0ff83087cb6", + "revision": "052f907cc35979930fda832a71aec0c2194e7e48", "modified": false, "content_sha256": "1b39e083738e6d10b515568bf7912241a1874cda150eb40908b92e7927d38b5f" } diff --git a/plugins/fxtr/skills/fxtr/provenance.json b/plugins/fxtr/skills/fxtr/provenance.json index 8c43586..f6c5e0e 100644 --- a/plugins/fxtr/skills/fxtr/provenance.json +++ b/plugins/fxtr/skills/fxtr/provenance.json @@ -2,10 +2,10 @@ "skill": "fxtr", "documentation": "https://docs.transluce.ai/fxtr", "packages": { - "fxtr": "0.0.1a4", - "behaviors": "0.0.1a3" + "fxtr": "0.1.0a0", + "behaviors": "0.0.1a4" }, - "revision": "c3782b939f40b8fbffa00c43c984c0ff83087cb6", + "revision": "052f907cc35979930fda832a71aec0c2194e7e48", "modified": false, - "content_sha256": "c4930667ed6ac4665628fab04edcd9a755516598d49cdd08199f1cb544779c7d" + "content_sha256": "4777ef0f3815d1b5efde11039f878d974f8221b50a1ca112904a4ad2e44472b5" } diff --git a/plugins/fxtr/skills/fxtr/references/reference/renderers.mdx b/plugins/fxtr/skills/fxtr/references/reference/renderers.mdx index ef9a08c..78c39fc 100644 --- a/plugins/fxtr/skills/fxtr/references/reference/renderers.mdx +++ b/plugins/fxtr/skills/fxtr/references/reference/renderers.mdx @@ -209,7 +209,9 @@ The viewer opens the detail beside the view, hands it the invocations its params (`target_model`, a dim of the step) and the params it declares (`proposition`; a link that leaves one out is a type error), and keeps it following the view. Type the detail's component with `ViewProps`: its `params` then binds each declared param (`params.proposition`), so -the view only checks the values (a string, one of a few names). `useLink` returns null where the +the view only checks the values (a string, one of a few names). A param's value is any JSON (a +string, a number, a list, an object), and nothing checks it before the view gets it, so the view +narrows it itself. `useLink` returns null where the viewer cannot open a view from here; the view then shows the detail in place. Declare a detail `within` the view that links to it when only that view does: its ID is named @@ -231,6 +233,30 @@ another step's output), name the location instead, with `to` (and `view`, for a the `params` it is opened at: ``. A link takes one or the other; with neither, it opens the entity alone. +### Params of an entity's views + +An entity's view (an `EntityPanel` renderer) can read params too, declared the same way: +`declareView(EntityPanel, "my_project.Transcript", "my-project.Transcript", { params: ["selector"] })`. +They are optional, since the viewer also opens the entity from links that bind none: the +component's `params.selector` may be absent. The main use is a **selector**, a value telling the +view which part of the entity to point out (a message to mark, a span to highlight), much like a +text fragment in a URL. fxtr gives selectors no meaning or shape of their own: a selector is a +param whose value the view decides, usually stored in a field of another entity (a judgment's +citation, beside the reference to the transcript it cites). A link carries it as `params`: + +```tsx + +``` + +A link with params opens the entity alone, and stays on it: the params are about that one +entity, so it doesn't follow the view it was opened from as a link with `at` does, and the panes +opened from it don't inherit them. The pane keeps every param the link passed and hands each of +the entity's views the ones it declares, whichever is shown, so two views of a type that read the +same param should agree on its shape. A pane's head shows a short param's value (a dim's key, say) +and a longer one's name alone, and on an entity's pane a × to clear it. Nothing checks that the +shape the producing step stores matches what the view reads: keep them the same, and have the view +ignore a value it can't use. + A view of the whole run (a chart across several steps) is a view of the root workflow. A detail that several views link to and that belongs to no one step can be a **page**: `declarePage(name, { params })` declares it and `page(decl, Component)` registers it. A page's @@ -363,6 +389,12 @@ updates without a restart. If a bundle fails to load in the browser, the viewer keeps the renderers it already loaded, or uses its defaults when none were loaded, and the side rail says why. +If fxtr says the bundle was built at another view API version, the renderers were built against +another fxtr's packages, usually a `views/vendor/fxtr/` copy from before you upgraded fxtr. A +release that changes the view API incompatibly bumps that version, and the viewer then refuses +bundles built at the old one. Refresh the copy, reinstall, and rebuild, as above, and fix any +renderer code that used an API the new `fxtr-view` no longer has. + ### Adding a package to a project To add a renderer package to a project without one, write these parts: