From 6ddf1b8b38c17b8a865614d4116ce0fcb7c6b5b8 Mon Sep 17 00:00:00 2001 From: Tejas Polakam Date: Mon, 2 Nov 2026 17:21:36 -0500 Subject: [PATCH 1/5] feat(episode): record episodes in the run index An episode called Task.run directly, so the run index never saw it and the explorer's run lists, which read only the index, never showed it: training episodes were reachable only by instance id. An episode now records itself as a runner records the runs it accepts (runner `episode`): RUNNING before Task.run, then COMPLETED, FAILED with the error, or CANCELED with the abandon reason once its teardown is done. `EpisodeSpec.start(run_group_id=...)` files it in a run group, so a trainer can show a group of rollouts together. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 3 +- src/agent_env/task/episode/run.py | 45 +++++++++++++++++++++++++++++- src/agent_env/task/episode/spec.py | 7 +++-- tst/unit/task/episode/run_test.py | 25 +++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5af2559e5..f34cdeb36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -372,7 +372,8 @@ orchestrators call it the same way. What callers rely on: in the proxy, which records the turn and checks it against the model calls the caller records. Its record is tagged `context.metadata["episode"]` and names the original task, so it isn't resumable with `start_step`. The episode tears down what its run deployed, and an abandoned one is recorded - `cancelled`. + `cancelled`. It records itself in the run index as a runner would (runner `episode`, in the run group + `start(run_group_id=...)` names), so the explorer lists it. ## Tests diff --git a/src/agent_env/task/episode/run.py b/src/agent_env/task/episode/run.py index 2b082cd69..6a0ce4d10 100644 --- a/src/agent_env/task/episode/run.py +++ b/src/agent_env/task/episode/run.py @@ -2,7 +2,10 @@ import asyncio import logging +import os +import socket from collections import deque +from datetime import datetime, timezone from typing import Any import httpx @@ -18,6 +21,8 @@ from agent_env.a2a_agent import protocol from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.providers.sandbox_providers.sandbox_provider import sandbox_request_headers_for_url +from agent_env.runner import store as run_store +from agent_env.runner.runner import RunRecord, RunStatus from agent_env.store.routing import namespace_routing, run_scope from agent_env.task.episode.proxy import InProcessProxy, ProxyClient, ProxyRefused from agent_env.task.store import record_task_cancelled @@ -249,10 +254,15 @@ async def _drive(self, running: asyncio.Event) -> None: resolves.""" running.set() with namespace_routing(): + _record_run(self._task, self.id, self._context.metadata) # before any await: start() returns it listed + failure: BaseException | None = None try: await self._task.run(context=self._context, instance_id=self.id) self._ran = True self._notify() + except BaseException as e: + failure = e + raise finally: self._tearing_down = True try: @@ -264,7 +274,19 @@ async def _drive(self, running: asyncio.Event) -> None: if self._abandoned is not None: await asyncio.to_thread(_record_cancelled, self._task.id, self.id, self._abandoned) finally: - del _live[self.id] + try: + await asyncio.to_thread(_record_ended, self._task.id, self.id, *self._outcome(failure)) + finally: + del _live[self.id] + + def _outcome(self, failure: BaseException | None) -> tuple[RunStatus, str | None]: + if self._abandoned is not None: + return RunStatus.CANCELED, self._abandoned + if isinstance(failure, asyncio.CancelledError): + return RunStatus.CANCELED, "the episode was cancelled" + if failure is not None: + return RunStatus.FAILED, f"{type(failure).__name__}: {failure}" + return RunStatus.COMPLETED, None async def _wait(self, changed: asyncio.Future[None]) -> None: """Until the run ends or what was ``changed`` when the caller last looked changes again.""" @@ -368,6 +390,27 @@ def _episode_of(context: TaskStepContext) -> Episode: return _live[context.metadata["episode"]["id"]] +def _record_run(task: Task, instance_id: str, metadata: dict[str, Any]) -> None: + """The episode in the run index as a runner records the runs it accepts, so the explorer lists it, in its run group + when ``start`` was given one; in the local stores for an ``@local`` task, as ``Task.run`` scopes its records.""" + now = datetime.now(timezone.utc).isoformat() + with run_scope(task.id): + run_store.insert_run(RunRecord( + run_id=_run_id(instance_id), runner="episode", task_id=task.id, task_version=task.version, + instance_id=instance_id, status=RunStatus.RUNNING, started_at_utc=now, + owner={"host": socket.gethostname(), "pid": os.getpid(), "seen": now}, + overrides={"metadata": {key: metadata[key] for key in ("episode", "run_group_id") if key in metadata}})) + + +def _record_ended(task_id: str, instance_id: str, status: RunStatus, error: str | None) -> None: + with run_scope(task_id): + run_store.mark_terminal(_run_id(instance_id), status, error=error) + + +def _run_id(instance_id: str) -> str: + return f"episode-{instance_id}" + + def _record_cancelled(task_id: str, instance_id: str, reason: str) -> None: """Marked where the run recorded its instance, as ``Task.run`` scopes it: an ``@local`` task's in the local stores.""" diff --git a/src/agent_env/task/episode/spec.py b/src/agent_env/task/episode/spec.py index 2eafc412b..516292ad1 100644 --- a/src/agent_env/task/episode/spec.py +++ b/src/agent_env/task/episode/spec.py @@ -56,14 +56,17 @@ def prompts(self) -> tuple[tuple[str, str | None], ...]: prompts.append((step.prompt, system_prompts[step.agent_name])) return tuple(prompts) - async def start(self, *, sandbox: str | None = None) -> Episode: + async def start(self, *, sandbox: str | None = None, run_group_id: str | None = None) -> Episode: """Start one run of the task, a new episode each call. It returns once the run's record exists, before anything is deployed; ``next_turn()`` is where the deploys and setup are waited for. ``sandbox`` is the provider its envs, agents, sandboxes and judge deploy on, as ``agent-env run --sandbox`` sets it. Its id is its run's, minted as ``agent-env run`` mints one, so what the run names after it stays in its task's - namespace.""" + namespace. The run is in the run index, which the explorer lists; ``run_group_id`` groups it with the other + runs given the same id.""" episode_id = new_instance_id(self._copy.id) metadata: dict[str, Any] = {"episode": {"id": episode_id, "trainer_agents": list(self.trainer_agents)}} + if run_group_id: + metadata["run_group_id"] = run_group_id if self._seed: metadata["seed"] = dict(self._seed) if sandbox: diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index a51290470..6a9fe3e4f 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -25,6 +25,7 @@ from agentenv_protocol.a2a_agent import proxy as proxy_module import agent_env.task.episode.run as episode_run +from agent_env.runner import store as run_store from agent_env.config import get_config from agent_env.config.runtime import Config from agent_env.env.env import DeployedEnv @@ -446,6 +447,30 @@ async def test_an_episode_abandoned_as_it_starts_is_recorded_cancelled_with_the_ await ask() +@pytest.mark.asyncio +async def test_an_episode_is_in_the_run_index_running_then_ended_in_the_group_it_was_started_in(teardowns): + played = await _spec(ITEMS, ALICE, TURN, REPLIES).start(run_group_id="rg-train-0") + run = run_store.get_run(f"episode-{played.id}") + assert (run.runner, run.task_id, run.task_version, run.instance_id, run.status) == ( + "episode", "dock", 4, played.id, "RUNNING") + assert run.overrides["metadata"] == {"episode": {"id": played.id, "trainer_agents": ["alice"]}, + "run_group_id": "rg-train-0"} + await (await played.next_turn()).end("Added a crate.") + await played.result() + abandoned = await _spec(ITEMS, ALICE, TURN, REPLIES).start() + await abandoned.abandon("the trainer moved on") + failed = await _spec({**ITEMS, "fail": True}, ALICE, TURN, REPLIES).start() + with pytest.raises(RuntimeError): + await failed.result() + + ended = {episode: run_store.get_run(f"episode-{episode.id}") for episode in (played, abandoned, failed)} + assert [(run.status, run.error) for run in ended.values()] == [ + ("COMPLETED", None), ("CANCELED", "the trainer moved on"), + ("FAILED", "RuntimeError: the env didn't come up")] + assert all(run.finished_at_utc for run in ended.values()) + assert "run_group_id" not in ended[abandoned].overrides["metadata"] + + @pytest.mark.asyncio async def test_an_episode_abandoned_mid_turn_drops_the_turn(teardowns): episode = await _spec(ITEMS, ALICE, TURN, REPLIES).start() From 5cdf1b1d24b34ba738a5ee684cf4268f73af61ba Mon Sep 17 00:00:00 2001 From: Tejas Polakam Date: Wed, 4 Nov 2026 14:08:06 -0500 Subject: [PATCH 2/5] fix(episode): record an episode's end as runners do, refuse cancelling it from the explorer, and keep a failed record write from leaking - An episode's run index status now follows `run_outcome`, the rule every runner uses: a step that fails without failing the task fails the run, rather than leaving it COMPLETED. Abandoned or cancelled stays CANCELED. - The explorer's cancel route refuses, with 409 and where to stop it, a run another runner records: the configured runner can't cancel an episode a trainer plays, so asking it silently did nothing. The UI shows the reason. - The run record is written inside the try whose finally takes the episode out of `_live`, so a write that fails is the run's error and leaves nothing behind. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/explorer/routers/runs.py | 12 ++++++++- .../ui/src/components/task-detail-page.tsx | 10 +++++++- src/agent_env/task/episode/run.py | 12 +++++---- tst/unit/explorer/explorer_app_test.py | 12 +++++++++ tst/unit/task/episode/run_test.py | 25 +++++++++++++++++++ 5 files changed, 64 insertions(+), 7 deletions(-) diff --git a/src/agent_env/explorer/routers/runs.py b/src/agent_env/explorer/routers/runs.py index f8da1d0f0..1113dd74d 100644 --- a/src/agent_env/explorer/routers/runs.py +++ b/src/agent_env/explorer/routers/runs.py @@ -222,7 +222,17 @@ async def start_runs(task_id: EntityId, body: StartRunsRequest | None = None) -> @router.post("/{task_id}/cancel-run") async def cancel_run(task_id: EntityId, workflow_id: str = Query(...)) -> dict: - canceled = await get_runner().cancel(workflow_id) + """Cancel a run of the configured runner. A run another process records, such as an episode a trainer plays, is + refused with 409: only the process running it can stop it, so asking this runner would silently do nothing.""" + from agent_env.runner import store as run_store + + runner = get_runner() + record = run_store.get_run(workflow_id) + if record is not None and record.runner != runner.type: + raise HTTPException(status_code=409, detail=( + f"run {workflow_id} is run by {record.runner!r}, not by this explorer's {runner.type!r} runner: stop it " + "where it runs (an episode, from the trainer that plays it)")) + canceled = await runner.cancel(workflow_id) return {"workflow_id": workflow_id, "canceled": canceled} diff --git a/src/agent_env/explorer/ui/src/components/task-detail-page.tsx b/src/agent_env/explorer/ui/src/components/task-detail-page.tsx index ea7bfa9be..064884c43 100644 --- a/src/agent_env/explorer/ui/src/components/task-detail-page.tsx +++ b/src/agent_env/explorer/ui/src/components/task-detail-page.tsx @@ -1087,7 +1087,15 @@ export function TaskDetailPage({ )}/cancel-run?workflow_id=${encodeURIComponent(workflowId)}`, { method: 'POST' }, ); - if (!res.ok) throw new Error(`Cancel failed (${res.status})`); + if (!res.ok) { + // A refusal says why (an episode is stopped by the trainer that plays it); show that, not just the status. + const body = (await res.json().catch(() => null)) as { detail?: unknown } | null; + throw new Error( + typeof body?.detail === 'string' + ? body.detail + : `Cancel failed (${res.status})`, + ); + } // Cancel endpoint flips Mongo to `cancelled` synchronously; refresh // so the row transitions without waiting for the next background poll. fetchRunGroups(); diff --git a/src/agent_env/task/episode/run.py b/src/agent_env/task/episode/run.py index 6a0ce4d10..ef732878e 100644 --- a/src/agent_env/task/episode/run.py +++ b/src/agent_env/task/episode/run.py @@ -22,7 +22,7 @@ from agent_env.a2a_agent.a2a_agent import A2AAgent from agent_env.providers.sandbox_providers.sandbox_provider import sandbox_request_headers_for_url from agent_env.runner import store as run_store -from agent_env.runner.runner import RunRecord, RunStatus +from agent_env.runner.runner import RunRecord, RunStatus, run_outcome from agent_env.store.routing import namespace_routing, run_scope from agent_env.task.episode.proxy import InProcessProxy, ProxyClient, ProxyRefused from agent_env.task.store import record_task_cancelled @@ -254,9 +254,11 @@ async def _drive(self, running: asyncio.Event) -> None: resolves.""" running.set() with namespace_routing(): - _record_run(self._task, self.id, self._context.metadata) # before any await: start() returns it listed failure: BaseException | None = None try: + # Before any await, so start() returns with the run listed; a write that fails is the run's error, and + # the finally below still takes the episode out of _live. + _record_run(self._task, self.id, self._context.metadata) await self._task.run(context=self._context, instance_id=self.id) self._ran = True self._notify() @@ -280,13 +282,13 @@ async def _drive(self, running: asyncio.Event) -> None: del _live[self.id] def _outcome(self, failure: BaseException | None) -> tuple[RunStatus, str | None]: + """How the run ended, by the rule every runner records a run's end by (``run_outcome``: a step that failed + without failing the task fails the run too), but cancelled when it was abandoned or cancelled.""" if self._abandoned is not None: return RunStatus.CANCELED, self._abandoned if isinstance(failure, asyncio.CancelledError): return RunStatus.CANCELED, "the episode was cancelled" - if failure is not None: - return RunStatus.FAILED, f"{type(failure).__name__}: {failure}" - return RunStatus.COMPLETED, None + return run_outcome(self._context, failure) async def _wait(self, changed: asyncio.Future[None]) -> None: """Until the run ends or what was ``changed`` when the caller last looked changes again.""" diff --git a/tst/unit/explorer/explorer_app_test.py b/tst/unit/explorer/explorer_app_test.py index f187a23d8..e41644e32 100644 --- a/tst/unit/explorer/explorer_app_test.py +++ b/tst/unit/explorer/explorer_app_test.py @@ -331,6 +331,18 @@ async def run(self, **kw): assert client.post(f"{base}/cancel-run", params={"workflow_id": single["workflow_id"]}).status_code == 200 +def test_a_run_another_runner_records_is_refused_cancelling_with_where_to_stop_it(client): + task_id = "@local/~/dock/crate" + run_store.insert_run(RunRecord(run_id="episode-abc", runner="episode", task_id=task_id, task_version=1, + instance_id="abc", status=RunStatus.RUNNING)) + + refused = client.post(f"/api/v1/tasks/{quote(task_id, safe='')}/cancel-run", params={"workflow_id": "episode-abc"}) + + assert refused.status_code == 409 + assert "run by 'episode'" in refused.json()["detail"] and "trainer that plays it" in refused.json()["detail"] + assert run_store.get_run("episode-abc").status == "RUNNING" + + def test_a_bundle_runs_namespaced_instance_id_is_one_encoded_segment(client): task_id, instance_id = "@local/~/triage/runs", "@local/~/triage/runs-7f3a9c2e" store = get_config().get_document_store() diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index 6a9fe3e4f..d46864dac 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -471,6 +471,31 @@ async def test_an_episode_is_in_the_run_index_running_then_ended_in_the_group_it assert "run_group_id" not in ended[abandoned].overrides["metadata"] +@pytest.mark.asyncio +async def test_an_episode_whose_step_fails_without_failing_the_task_is_recorded_failed_as_a_runner_would(teardowns): + spare = {"id": "spare", "type": "episode_test_env", "env_id": "spare", "fail": True, "fail_task_on_error": False} + episode = await _spec(ITEMS, ALICE, TURN, REPLIES, spare).start() + await (await episode.next_turn()).end("Added a crate.") + + assert await episode.result() == {"replies": 1.0} + run = run_store.get_run(f"episode-{episode.id}") + assert run.status == "FAILED" and "the env didn't come up" in run.error + + +@pytest.mark.asyncio +async def test_a_run_record_that_cant_be_written_is_the_runs_error_and_leaves_no_live_episode(teardowns, monkeypatch): + def refused(record): + raise RuntimeError("the run index is down") + + monkeypatch.setattr(episode_run.run_store, "insert_run", refused) + episode = await _spec(ITEMS, ALICE, TURN, REPLIES).start() + + with pytest.raises(RuntimeError, match="the run index is down"): + await episode.next_turn() + assert episode.id not in episode_run._live + assert len(teardowns.finished) == 1 + + @pytest.mark.asyncio async def test_an_episode_abandoned_mid_turn_drops_the_turn(teardowns): episode = await _spec(ITEMS, ALICE, TURN, REPLIES).start() From 167a7828493881432c2ac0c07bc356e9418c0082 Mon Sep 17 00:00:00 2001 From: Tejas Polakam Date: Fri, 6 Nov 2026 11:47:29 -0500 Subject: [PATCH 3/5] fix(episode): keep an episode's seed in its run record The explorer reads a run's seed from `overrides.metadata.seed`, so an episode's run group listed `seed: null` for a seeded episode. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task/episode/run.py | 3 ++- tst/unit/task/episode/run_test.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task/episode/run.py b/src/agent_env/task/episode/run.py index ef732878e..4f9e594ba 100644 --- a/src/agent_env/task/episode/run.py +++ b/src/agent_env/task/episode/run.py @@ -401,7 +401,8 @@ def _record_run(task: Task, instance_id: str, metadata: dict[str, Any]) -> None: run_id=_run_id(instance_id), runner="episode", task_id=task.id, task_version=task.version, instance_id=instance_id, status=RunStatus.RUNNING, started_at_utc=now, owner={"host": socket.gethostname(), "pid": os.getpid(), "seen": now}, - overrides={"metadata": {key: metadata[key] for key in ("episode", "run_group_id") if key in metadata}})) + overrides={"metadata": {key: metadata[key] for key in ("episode", "run_group_id", "seed") + if key in metadata}})) def _record_ended(task_id: str, instance_id: str, status: RunStatus, error: str | None) -> None: diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index d46864dac..8e18231d9 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -449,12 +449,12 @@ async def test_an_episode_abandoned_as_it_starts_is_recorded_cancelled_with_the_ @pytest.mark.asyncio async def test_an_episode_is_in_the_run_index_running_then_ended_in_the_group_it_was_started_in(teardowns): - played = await _spec(ITEMS, ALICE, TURN, REPLIES).start(run_group_id="rg-train-0") + played = await _spec(ITEMS, ALICE, TURN, REPLIES, seed={"week": "3"}).start(run_group_id="rg-train-0") run = run_store.get_run(f"episode-{played.id}") assert (run.runner, run.task_id, run.task_version, run.instance_id, run.status) == ( "episode", "dock", 4, played.id, "RUNNING") assert run.overrides["metadata"] == {"episode": {"id": played.id, "trainer_agents": ["alice"]}, - "run_group_id": "rg-train-0"} + "run_group_id": "rg-train-0", "seed": {"week": "3"}} await (await played.next_turn()).end("Added a crate.") await played.result() abandoned = await _spec(ITEMS, ALICE, TURN, REPLIES).start() @@ -468,7 +468,7 @@ async def test_an_episode_is_in_the_run_index_running_then_ended_in_the_group_it ("COMPLETED", None), ("CANCELED", "the trainer moved on"), ("FAILED", "RuntimeError: the env didn't come up")] assert all(run.finished_at_utc for run in ended.values()) - assert "run_group_id" not in ended[abandoned].overrides["metadata"] + assert ended[abandoned].overrides["metadata"].keys() == {"episode"} @pytest.mark.asyncio From 304ead5f8285215c7f3c127000fdf0ae6c6f326f Mon Sep 17 00:00:00 2001 From: Tejas Polakam Date: Tue, 17 Nov 2026 10:57:10 -0500 Subject: [PATCH 4/5] feat(training): train with Tinker on a bundle's tasks (#458) Co-authored-by: Claude Opus 5.5 --- .github/workflows/tinker.yml | 65 +++ AGENTS.md | 12 + Makefile | 8 +- pyproject.toml | 7 +- src/agent_env/training/_episodes.py | 86 +++ src/agent_env/training/_openenv.py | 9 +- src/agent_env/training/tinker.py | 506 ++++++++++++++++++ tst/tinker/__init__.py | 0 tst/tinker/conftest.py | 6 + tst/tinker/episode_env_test.py | 260 +++++++++ .../test_optional_extras_stay_optional.py | 1 + tst/unit/training/episodes_test.py | 52 ++ tst/util/capabilities.py | 4 + uv.lock | 8 +- 14 files changed, 1015 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/tinker.yml create mode 100644 src/agent_env/training/_episodes.py create mode 100644 src/agent_env/training/tinker.py create mode 100644 tst/tinker/__init__.py create mode 100644 tst/tinker/conftest.py create mode 100644 tst/tinker/episode_env_test.py create mode 100644 tst/unit/training/episodes_test.py diff --git a/.github/workflows/tinker.yml b/.github/workflows/tinker.yml new file mode 100644 index 000000000..d6babb8e8 --- /dev/null +++ b/.github/workflows/tinker.yml @@ -0,0 +1,65 @@ +name: Tinker contract + +# The Tinker helper's episodes on tinker-cookbook itself (tst/tinker): EpisodeEnv played as the cookbook's rollouts step +# a MessageEnv, on the Episode API's fake steps with a stand-in renderer, so nothing samples and no Tinker key is needed, +# on tinker-cookbook as released and on its main branch. tinker-cookbook and CPU torch are installed over the dev +# environment. It runs on pull requests that touch the helper, the Episode API or the tier, weekly, and on demand; it +# isn't a required check. + +on: + pull_request: + paths: + - "src/agent_env/training/**" + - "src/agent_env/task/episode/**" + - "src/agent_env/openenv/reward.py" + - "tst/tinker/**" + - "tst/unit/training/**" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/tinker.yml" + schedule: + - cron: "17 8 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: tinker-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + +jobs: + episodes: + name: episodes (tinker-cookbook ${{ matrix.cookbook }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - cookbook: "released" + requirement: "tinker-cookbook" + - cookbook: "main" + requirement: "tinker-cookbook@git+https://github.com/thinking-machines-lab/tinker-cookbook@main" + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Install from uv.lock with the tinker extra + run: | + python -m pip install --quiet -r .github/ci-requirements.txt + uv sync --locked --extra dev --extra tinker --python "$(command -v python)" + - name: Install tinker-cookbook and CPU torch + run: | + uv pip install --python .venv/bin/python ${{ matrix.requirement }} torch \ + --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match + .venv/bin/python -c "from importlib.metadata import version; print('tinker-cookbook', version('tinker-cookbook'), 'tinker', version('tinker'), 'torch', version('torch'))" + - name: Tinker tier + run: .venv/bin/python -m pytest tst/tinker -rA --timeout=900 --junitxml=junit-tinker.xml + - name: No skips + if: always() + run: .venv/bin/python .github/scripts/check_skip_policy.py junit-tinker.xml diff --git a/AGENTS.md b/AGENTS.md index f34cdeb36..210df24f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -154,6 +154,10 @@ Shared: in-process `Server`. - `_tools.py`: turns an env's tools into typed, documented Python methods from their JSON Schema, as transformers' `get_json_schema` reads them. +- `_episodes.py`, what a helper needs without its trainer installed: `typed_arguments` types a tool call's arguments by + the tool's schema (a renderer that writes them as untyped text reads digits as numbers), `next_prompt` writes a later + turn's prompt as every helper's model reads it (a system prompt the turn changes, as text before it), and `RunGroup` + with `TrainingSteps` names the run group a helper files a group's episodes in, `/train|eval/step-`. - `_chat.py`, the rows and episode the Miles and slime helpers share: - `write_rows` and its `command` write a server's tasks as `.jsonl` rows of chat `messages` and `metadata` (the episode's split, index, task id and tools, and the server it plays on with its deadlines and turn cap). @@ -168,6 +172,14 @@ Per trainer: - `trl.py` (`trl` extra): `AgentEnv(url)` over a server `openenv/` makes, with `dataset` (a row per task) and `factory` (TRL's `environment_factory`: a generated class per toolset whose methods are the tools, a background `reset` and an owned `get_reward`); `AgentEnv.local(bundle)` serves the bundle in process. +- `tinker.py` (`tinker` extra, installed into tinker-cookbook's environment): `AgentEnvDatasetBuilder` is `rl.train`'s + dataset builder, each rollout an episode played in process on the Episode API rather than through a server, since + Tinker samples itself. The turn's tools are rendered by the cookbook's renderer, each sampled message is recorded before + its calls run, a reply without a call ends the turn, and the reward follows `openenv/reward.py`. A later turn's new + system prompt comes before its prompt (`next_prompt`), a turn that offers other tools than the first fails its + rollout, and a task that gives the agent skills is left out. A rollout the cookbook cuts + is scored as it is cut, which frees its slot under `max_concurrent_episodes`; a failed episode raises for the + cookbook's retry; each group is filed in the run index by `RunGroup`. - `harness.py` (`harness` extra): trains a bundle's own agents with TRL's harness rollouts (`HarnessRolloutWorker` with `harness_adapter=None`). `AgentEnvSessionFactory` runs each rollout's task as `agent-env run` does: the solver, the agent a step prompts, reaches the trainer's vLLM through OpenEnv's capture proxy, which records the tokens TRL trains diff --git a/Makefile b/Makefile index 99fffb01e..a72d76c6b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install lint unit-test int-test int-test-fast int-test-slow installer-test trl-test nemo-gym-test verl-test skyrl-test miles-test slime-test clean-install-test test clean +.PHONY: help install lint unit-test int-test int-test-fast int-test-slow installer-test trl-test nemo-gym-test verl-test skyrl-test miles-test slime-test tinker-test clean-install-test test clean VENV := .venv VENV_BIN := $(abspath $(VENV))/bin @@ -97,6 +97,12 @@ miles-test: ## Play rows through Miles' rollout and session server (needs Miles, slime-test: ## Play rows through the slime helper on slime's OpenAI adapter (needs slime, SGLang and Megatron-LM, CPU torch, and the Hub) $(PYTHON) -m pytest tst/slime --timeout=900 $(PYTEST_ARGS) +# The Tinker tier plays episodes through agent_env.training.tinker on tinker-cookbook itself, with a stand-in renderer, +# so nothing samples. tinker-cookbook and torch aren't in the dev extra: install them first, as +# .github/workflows/tinker.yml does. +tinker-test: ## Play episodes through the Tinker helper on tinker-cookbook itself (needs tinker-cookbook and CPU torch) + $(PYTHON) -m pytest tst/tinker --timeout=900 $(PYTEST_ARGS) + # The clean-install gate CI runs, on the same Python. The script gives every step an allowlisted # environment, so nothing in this shell reaches the install it checks. A failed run keeps its folder. clean-install-test: ## Build both distributions, install them clean from public PyPI and run hello twice (needs python3.11, uv, git) diff --git a/pyproject.toml b/pyproject.toml index d8c9a25a6..4650c2366 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,6 +104,11 @@ miles = [ slime = [ "agentenv-framework[openenv]", ] +# agent_env.training.tinker: tinker-cookbook's rl.train on a bundle's tasks, each rollout an episode played in process; +# openenv for the reward rules build_app applies. tinker-cookbook, Tinker and torch are the trainer's to pin. +tinker = [ + "agentenv-framework[openenv]", +] # The AWS store backends (S3, Secrets Manager, DynamoDB, ECR), selected by impl pointer in config.toml. aws = [ "boto3>=1.35.0", @@ -152,7 +157,7 @@ coreweave = [ "cwsandbox>=1.19.6,<2", # Reject exec streams that close without an exit status. ] dev = [ - "agentenv-framework[aws,boat,boxd,coreweave,daytona,gcp,harness,openenv,sail,skyrl,tensorlake,tenki,trl,vercel,verl]", # exercise portable optional providers and stores in CI + "agentenv-framework[aws,boat,boxd,coreweave,daytona,gcp,harness,openenv,sail,skyrl,tensorlake,tenki,tinker,trl,vercel,verl]", # exercise portable optional providers and stores in CI "fastmcp>=3.4,<4", # the standalone-fastmcp mount path of the protocol; <4 keeps mcp 1.x "Pillow>=10.4", # the protocol's computer extra, whose tests the unit tier runs "av>=17", # the protocol's computer extra, whose tests the unit tier runs and which the recording tests decode with diff --git a/src/agent_env/training/_episodes.py b/src/agent_env/training/_episodes.py new file mode 100644 index 000000000..e601f0f9c --- /dev/null +++ b/src/agent_env/training/_episodes.py @@ -0,0 +1,86 @@ +"""What a trainer's adapter needs without the trainer installed: a tool call's arguments typed by its tool's schema, +a later turn's prompt as the model reads it, and the run group a trainer files its episodes in.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any, Literal + + +_JSON_TYPES = {bool: "boolean", int: "integer", float: "number", type(None): "null"} + + +def typed_arguments(arguments: Any, schema: Mapping[str, Any]) -> Any: + """``arguments`` with each scalar whose type the tool's schema doesn't allow, but a string, as the string it was + written as. Renderers that write parameters as untyped text (Qwen3.5's ``19122``) parse + each with ``json.loads``, reading digits as a number; Qwen's own parser types values by the schema, as this does.""" + if not isinstance(arguments, dict): + return arguments + properties = schema.get("properties") or {} + typed = dict(arguments) + for name, value in arguments.items(): + allowed = _allowed_types(properties.get(name) or {}) + kind = _JSON_TYPES.get(type(value)) + widened = kind == "integer" and "number" in allowed + if kind is not None and "string" in allowed and kind not in allowed and not widened: + typed[name] = json.dumps(value) + return typed + + +def next_prompt(prompt: str, system_prompt: str | None, in_effect: str | None) -> tuple[str, str | None]: + """A later turn's prompt as every trainer's model reads it, and the system prompt in effect after it. A trainer + plays an episode as one conversation, whose system prompt is written once, at its start; so a turn that changes the + system prompt has the new one written before its prompt, as text. A turn that sets none, or the one in effect, + is its prompt alone.""" + if not system_prompt or system_prompt == in_effect: + return prompt, in_effect + return f"{system_prompt}\n\n{prompt}", system_prompt + + +def _allowed_types(schema: Mapping[str, Any]) -> set[str]: + declared = schema.get("type") + allowed = set(declared) if isinstance(declared, list) else {declared} if declared else set() + for option in schema.get("anyOf") or schema.get("oneOf") or []: + allowed |= _allowed_types(option) + return allowed + + +def run_group_id(run: str, phase: Literal["train", "eval"], step: int) -> str: + """The run group a trainer files a group of episodes in: its training run, whether they were played to train or + to evaluate, and the training step whose policy played them. Every trainer names its groups so, and the explorer + lists a task's runs by them.""" + return f"{run}/{phase}/step-{step:04d}" + + +class TrainingSteps: + """The training step a trainer's batches are played at: a training batch's index, and for an evaluation the step + after the last training batch handed out, for a trainer that, as ``rl.train`` does, evaluates a step's policy + before it hands out that step's batch and once more after the last.""" + + def __init__(self) -> None: + self._last_trained: int | None = None + + def trained(self, index: int) -> int: + self._last_trained = index + return index + + def evaluated(self) -> int: + return 0 if self._last_trained is None else self._last_trained + 1 + + +@dataclass(frozen=True) +class RunGroup: + """Where a trainer files a group of episodes: under the training batch ``trained`` names, fixed when the batch is + handed out, or, for an evaluation, under the step it is played at, read each time it is played, since a trainer + may make its evaluation's groups once and play them at every evaluation, as ``rl.train`` does.""" + + run: str + steps: TrainingSteps + trained: int | None = None + + def id(self) -> str: + if self.trained is not None: + return run_group_id(self.run, "train", self.trained) + return run_group_id(self.run, "eval", self.steps.evaluated()) diff --git a/src/agent_env/training/_openenv.py b/src/agent_env/training/_openenv.py index 8b96ae404..e82838c1c 100644 --- a/src/agent_env/training/_openenv.py +++ b/src/agent_env/training/_openenv.py @@ -24,6 +24,7 @@ from agent_env.openenv.environment import END_TURN as END_TURN_TOOL from agent_env.openenv.environment import TIMEOUT_METADATA, Toolset from agent_env.task.teardown import TERMINATE_TIMEOUT_SECONDS +from agent_env.training._episodes import next_prompt FAILED = "The environment failed to answer, so this episode won't be scored." ENDED = "The episode has ended." @@ -215,11 +216,9 @@ def _answer(self, rollout: Rollout, name: str, result: Any) -> str: return ENDED answer = observation.get("result") or {} if name == END_TURN: - prompt, system = answer.get("prompt") or "", answer.get("system_prompt") or None - if system is None or system == rollout.system: - return prompt - rollout.system = system - return f"{system}\n\n{prompt}" + prompt, rollout.system = next_prompt(answer.get("prompt") or "", answer.get("system_prompt"), + rollout.system) + return prompt return answer["data"] if isinstance(answer.get("data"), str) else json.dumps(answer) async def _fail(self, rollout: Rollout, why: str) -> None: diff --git a/src/agent_env/training/tinker.py b/src/agent_env/training/tinker.py new file mode 100644 index 000000000..0f36e59ed --- /dev/null +++ b/src/agent_env/training/tinker.py @@ -0,0 +1,506 @@ +"""Tinker's RL on a bundle's tasks: tinker-cookbook's ``rl.train``, each rollout one episode of a task +(``agent_env.task.EpisodeSpec``) whose agent the model being trained plays. ``AgentEnvDatasetBuilder`` is the +``dataset_builder`` of ``rl.train.Config``: each batch is groups of episodes, a group playing one task, and every +episode is a ``Task.run`` of its own that the task's verifiers score as ``agent-env run`` would. + +The model reads the turn's system prompt, its tools (the envs' tools, and ``bash`` when its agent works in a sandbox) +as the cookbook's renderer writes them, and the turn's prompt. Each sampled message is recorded on the turn before its +tool calls run; a message without tool calls ends the turn, and the run's next turn, if it has one, comes as the next +user message. Once the run ends, the rollout's reward is the task's reward of its scores by the rules ``build_app`` +applies (``agent_env.openenv.reward``: the score at the step id ``reward`` names, or the task's one score), and each +score is a metric ``score/``. + +The system prompt and the tools are written once, at the start of the conversation, as the cookbook's renderers write +them. A later turn that changes the system prompt has the new one written before its prompt, as every trainer's model +reads it (``agent_env.training._episodes.next_prompt``); a turn that offers other tools than the first fails its +rollout; and a task that gives the agent skills, which no trainer shows its model yet, is left out (``playable``). + +An episode lives on the event loop that started it, so every rollout runs on ``rl.train``'s: set no rollout executor. +A rollout the cookbook ends itself (a structural parse failure, the context or token limit) is scored the moment it is +cut (``_ScoredWhenCut``): its turn ends with no reply, the verifiers score where the episode got to, that score is added +to the cookbook's own reward for the cut, ``failed_reward``, and the episode's slot under ``max_concurrent_episodes`` is +freed. The metric ``cut`` is 1 for such a rollout and 0 for the rest.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import json +import logging +import random +from collections.abc import Callable, Sequence +from pathlib import Path +from typing import Any, ClassVar + +try: + import chz + from tinker_cookbook import tokenizer_utils + from tinker_cookbook.renderers import get_renderer + from tinker_cookbook.renderers.base import Message, Renderer, ToolCall, ToolSpec, get_text_content + from tinker_cookbook.rl.message_env import EnvFromMessageEnv, MessageEnv, MessageStepResult + from tinker_cookbook.rl.types import (Action, ActionExtra, Env, EnvGroupBuilder, InitialObservationOverflow, + RLDataset, RLDatasetBuilder, StepResult) +except ModuleNotFoundError as exc: + if (exc.name or "").split(".")[0] not in ("chz", "tinker", "tinker_cookbook"): + raise + raise ImportError("agent_env.training.tinker runs in tinker-cookbook's environment: install tinker-cookbook and " + "'agentenv-framework[tinker]' there") from exc + +from agent_env.bundle import BundleKind, parse_bundle +from agent_env.bundle.run import prepare_bundle +from agent_env.store.routing import namespace_routing +from agent_env.task import Episode, EpisodeError, EpisodeSpec, Task, Turn +from agent_env.openenv.reward import reward_of +from agent_env.training._episodes import RunGroup, TrainingSteps, next_prompt, typed_arguments + +logger = logging.getLogger(__name__) + +Reward = Callable[[dict[str, float]], float | None] + +#: Per event loop, the episodes that may be under way at once, by limit: a semaphore can't be pickled with a builder. +_SLOTS: dict[tuple[int, int], asyncio.Semaphore] = {} + + +def _slots(limit: int) -> asyncio.Semaphore: + key = (id(asyncio.get_running_loop()), limit) + if key not in _SLOTS: + _SLOTS[key] = asyncio.Semaphore(limit) + return _SLOTS[key] + + +class EpisodeEnv(MessageEnv): + """One episode of ``spec``'s task as a cookbook ``MessageEnv``: started at the first observation, played turn by + turn, rewarded from its scores once the run ends. ``max_turns`` bounds the model's messages; at the bound the turn + ends with what the model last said, and every later turn with no reply, so the verifiers score where it got to. + + The episode holds a slot under ``max_concurrent_episodes`` from its start until its run is scored or abandoned, + since a run is containers of its own. The system prompt and the tools are the first turn's, written once; a later + turn's new system prompt comes before its prompt (``next_prompt``).""" + + #: The most of an assistant message, a call's arguments or a tool's answer a step logs, in characters. + LOGGED_CHARS: ClassVar[int] = 2000 + + def __init__(self, name: str, spec: EpisodeSpec, *, renderer: Renderer, reward: Reward, model_name: str, + sandbox: str | None, max_turns: int, tool_timeout: float, max_concurrent_episodes: int, + run_group_id: str): + self._name = name + self._run_group_id = run_group_id + self._spec = spec + self._renderer = renderer + self._reward = reward + self._model_name = model_name + self._sandbox = sandbox + self._max_turns = max_turns + self._tool_timeout = tool_timeout + self._max_concurrent = max_concurrent_episodes + self.history: list[Message] = [] + self._episode: Episode | None = None + self._turn: Turn | None = None + self._tools: Any = None + self._specs: list[ToolSpec] | None = None + self._system: str | None = None + self._holding = False + self._scored = False + self._messages = 0 + self._tool_calls = 0 + self._tool_errors = 0 + + @property + def scored(self) -> bool: + """Whether the episode's run has ended and been rewarded.""" + return self._scored + + async def initial_observation(self) -> list[Message]: + await _slots(self._max_concurrent).acquire() + self._holding = True + try: + self._episode = await self._spec.start(sandbox=self._sandbox, run_group_id=self._run_group_id) + turn = await self._episode.next_turn() + if turn is None: + raise EpisodeError(f"episode {self._episode.id} ended before its first turn") + await self._open(turn) + self._system = turn.system_prompt + prefix = self._renderer.create_conversation_prefix_with_tools(tools=self._specs or [], + system_prompt=self._system or "") + self.history = [*prefix, {"role": "user", "content": turn.prompt}] + except BaseException: + await self.close() # a failed episode gives its slot back now, not when its group is cleaned up + raise + return self.history + + async def step(self, message: Message) -> MessageStepResult: + try: + return await self._step(message) + except BaseException: + await self.close() + raise + + async def _step(self, message: Message) -> MessageStepResult: + self.history.append(message) + self._messages += 1 + text = get_text_content(message) + calls = [(call, call.id or f"call_{self._messages}_{n}", typed_arguments(_arguments(call), self._schema(call))) + for n, call in enumerate(message.get("tool_calls") or [])] + # A call whose arguments aren't an object can't run, so it isn't recorded: the turn runs every recorded call. + await self._turn.record_model_call( + _chat_message(text, [call for call in calls if isinstance(call[2], dict)]), model=self._model_name) + logs: dict[str, str | int | float] = {"task": self._name, "episode": self._episode.id, + "assistant": _clipped(text, self.LOGGED_CHARS)} + for n, (call, call_id, arguments) in enumerate(calls): + if isinstance(arguments, dict): + result = await self._run(call, call_id, arguments) + else: + self._tool_errors += 1 + result = _tool_message(call, call_id, "error: the arguments aren't a JSON object") + self.history.append(result) + logs[f"tool_call_{n}"] = (f"{call.function.name}({_clipped(json.dumps(arguments), self.LOGGED_CHARS)}) -> " + f"{_clipped(str(result['content']), self.LOGGED_CHARS)}") + unparsed = message.get("unparsed_tool_calls") or [] + for each in unparsed: + self._tool_errors += 1 + self.history.append({"role": "user", "content": f"Your tool call couldn't be parsed ({each.error}). " + "Call the tool again, with JSON arguments."}) + if (calls or unparsed) and self._messages < self._max_turns: + return MessageStepResult(reward=0.0, episode_done=False, next_messages=self.history, logs=logs) + result = await self._end_turn("" if calls else text) + result.logs = {**logs, **result.logs} + return result + + async def close(self) -> None: + """Abandon the episode unless its run has ended, and give back its slot. Safe to call more than once.""" + try: + if self._episode is not None and not self._episode.finished: + await self._episode.abandon("its rollout ended before the episode did") + finally: + self._release() + + async def _open(self, turn: Turn) -> None: + """Make ``turn`` the one being played, refusing what the conversation can't show the model: skills, and tools + other than the first turn's, which the renderer wrote at its start.""" + if turn.skills: + raise EpisodeError(f"turn {turn.step_id!r} gives the agent skills {[s.name for s in turn.skills]}, which " + "no trainer shows its model yet") + self._turn = turn + self._tools = await turn.tools(timeout=self._tool_timeout).__aenter__() + specs: list[ToolSpec] = [{"name": tool.name, "description": tool.description or "", + "parameters": tool.inputSchema or {"type": "object", "properties": {}}} + for tool in self._tools.list()] + if self._specs is not None and specs != self._specs: + raise EpisodeError(f"turn {turn.step_id!r} offers other tools than the episode's first turn, and the " + "renderer writes the tools once, at the start of the conversation") + self._specs = specs + + def _schema(self, call: ToolCall) -> dict: + return next((spec["parameters"] for spec in self._specs or [] if spec["name"] == call.function.name), {}) + + async def _run(self, call: ToolCall, call_id: str, arguments: dict) -> Message: + self._tool_calls += 1 + name = call.function.name + offered = [spec["name"] for spec in self._specs or []] + if name not in offered: + self._tool_errors += 1 + return _tool_message(call, call_id, f"error: there is no tool {name!r}; the tools are {offered}") + result = await self._tools.call(name, arguments) + self._tool_errors += bool(result.isError) + text = "".join(_text(part) for part in result.content) + return _tool_message(call, call_id, f"error: {text}" if result.isError else text) + + async def score_cut(self) -> tuple[float, dict[str, float]]: + """The verifiers' reward for an episode whose rollout the cookbook cut, called the moment it is cut + (``_ScoredWhenCut``): its open turn ends with no reply, its run is scored and its slot freed. Nothing for an + episode ``step`` already rewarded.""" + if self._scored: + return 0.0, {"cut": 0.0} + try: + if self._turn is not None: + await self._turn.end("") + reward, metrics, _ = await self._finish() + except BaseException: + await self.close() + raise + return reward, {**metrics, "cut": 1.0} + + async def _end_turn(self, reply: str) -> MessageStepResult: + await self._turn.end(reply) + while (turn := await self._episode.next_turn()) is not None: + if self._messages < self._max_turns: + await self._open(turn) + prompt, self._system = next_prompt(turn.prompt, turn.system_prompt, self._system) + self.history.append({"role": "user", "content": prompt}) + return MessageStepResult(reward=0.0, episode_done=False, next_messages=self.history) + await turn.end("") + reward, metrics, scores = await self._finish() + return MessageStepResult(reward=reward, episode_done=True, next_messages=self.history, metrics=metrics, + logs={"scores": json.dumps(scores)}) + + async def _finish(self) -> tuple[float, dict[str, float], dict[str, float]]: + """End every turn the run has left with no reply, and reward the scores the run records.""" + while (turn := await self._episode.next_turn()) is not None: + await turn.end("") + scores = await self._episode.result() + self._release() + self._scored = True + reward = self._reward(scores) + if reward is None: + raise EpisodeError(f"episode {self._episode.id} recorded no score to make a reward of: {scores}") + metrics = {f"score/{step}": score for step, score in scores.items()} + metrics |= {"messages": self._messages, "tool_calls": self._tool_calls, "tool_errors": self._tool_errors, + "cut": 0.0} + return reward, metrics, scores + + def _release(self) -> None: + if self._holding: + self._holding = False + _slots(self._max_concurrent).release() + + +def _arguments(call: ToolCall) -> Any: + try: + return json.loads(call.function.arguments or "{}") + except json.JSONDecodeError: + return None + + +def _clipped(text: str, limit: int) -> str: + return text if len(text) <= limit else f"{text[:limit]}… ({len(text) - limit} more characters)" + + +def _chat_message(text: str, calls: list[tuple[ToolCall, str, dict]]) -> dict[str, Any]: + """The model's message as an OpenAI chat message, with the tool calls that will run.""" + message: dict[str, Any] = {"role": "assistant", "content": text} + if calls: + message["tool_calls"] = [{"id": call_id, "type": "function", + "function": {"name": call.function.name, "arguments": json.dumps(arguments)}} + for call, call_id, arguments in calls] + return message + + +def _tool_message(call: ToolCall, call_id: str, content: str) -> Message: + return {"role": "tool", "content": content, "tool_call_id": call_id, "name": call.function.name} + + +def _text(part: Any) -> str: + kind = part.get("type") if isinstance(part, dict) else getattr(part, "type", None) + if kind == "text": + return part.get("text", "") if isinstance(part, dict) else part.text + return f"[{kind} content]" + + +class _ScoredWhenCut(EnvFromMessageEnv): + """The cookbook's token-level ``Env`` over an ``EpisodeEnv``, scoring a rollout the moment it cuts it. + + ``EnvFromMessageEnv`` ends a rollout itself, without calling the ``EpisodeEnv`` behind it, when a message runs past + the token limit, the response can't be parsed, or the conversation outgrows ``max_trajectory_tokens``. The episode + is then left with its turn open, still holding its containers and its slot under ``max_concurrent_episodes``. Left + until its group is done, such episodes can hold every slot while their groups' other rollouts wait for one, and + training stops. So the cut is scored here, where it happens, which frees the slot at once.""" + + async def initial_observation(self): + observation = await super().initial_observation() + if isinstance(observation, InitialObservationOverflow): # the prompt alone is over the token budget + reward, metrics = await self.message_env.score_cut() + observation = dataclasses.replace(observation, reward=observation.reward + reward, + metrics={**observation.metrics, **metrics}) + return observation + + async def step(self, action: Action, *, extra: ActionExtra | None = None) -> StepResult: + result = await super().step(action, extra=extra) + if result.episode_done and not self.message_env.scored: # the cookbook ended it, not the episode + reward, metrics = await self.message_env.score_cut() + result = dataclasses.replace(result, reward=result.reward + reward, metrics={**result.metrics, **metrics}) + return result + + +class EpisodeGroupBuilder(EnvGroupBuilder): + """``group_size`` episodes of one task, each its own run, filed in ``run_group``, which the explorer shows + together. ``make_envs`` may be called again for a retry, and ``cleanup`` abandons every episode still under way.""" + + def __init__(self, name: str, spec: EpisodeSpec, reward: Reward, settings: _Settings, group_size: int, + run_group: RunGroup): + self.name = name + self.run_group = run_group + self.spec = spec + self.reward = reward + self.settings = settings + self.group_size = group_size + self._made: list[EpisodeEnv] = [] + + async def make_envs(self) -> Sequence[Env]: + renderer = self.settings.renderer() + run_group_id = self.run_group.id() + envs = [] + for _ in range(self.group_size): + episode_env = EpisodeEnv(self.name, self.spec, renderer=renderer, reward=self.reward, + model_name=self.settings.model_name, sandbox=self.settings.sandbox, + max_turns=self.settings.max_turns, tool_timeout=self.settings.tool_timeout, + max_concurrent_episodes=self.settings.max_concurrent_episodes, + run_group_id=run_group_id) + self._made.append(episode_env) + envs.append(_ScoredWhenCut(renderer, episode_env, failed_parse_reward=self.settings.failed_reward, + context_overflow_reward=self.settings.failed_reward, + max_trajectory_tokens=self.settings.max_trajectory_tokens)) + return envs + + async def cleanup(self) -> None: + made, self._made = self._made, [] + for result in await asyncio.gather(*(env.close() for env in made), return_exceptions=True): + if isinstance(result, BaseException): + logger.warning("task %s: an episode didn't close: %s", self.name, result) + + def logging_tags(self) -> list[str]: + return [self.settings.tag] + + +class _Settings: + """What every group of a dataset shares.""" + + def __init__(self, *, model_name: str, renderer_name: str, sandbox: str | None, max_turns: int, + tool_timeout: float, max_concurrent_episodes: int, max_trajectory_tokens: int | None, + failed_reward: float, tag: str): + self.model_name = model_name + self.renderer_name = renderer_name + self.sandbox = sandbox + self.max_turns = max_turns + self.tool_timeout = tool_timeout + self.max_concurrent_episodes = max_concurrent_episodes + self.max_trajectory_tokens = max_trajectory_tokens + self.failed_reward = failed_reward + self.tag = tag + + def renderer(self) -> Renderer: + return get_renderer(self.renderer_name, tokenizer_utils.get_tokenizer(self.model_name), + model_name=self.model_name) + + +class EpisodeDataset(RLDataset): + """A split's tasks in batches of ``groups_per_batch`` groups, shuffled anew each of ``epochs`` passes; a group's + episodes are filed under the run, whether they train or evaluate, and the training step that plays them.""" + + def __init__(self, tasks: list[tuple[str, EpisodeSpec, Reward]], settings: _Settings, *, group_size: int, + groups_per_batch: int, epochs: int, seed: int, shuffle: bool, run_name: str, + steps: TrainingSteps, trains: bool): + self._run_name = run_name + self._steps = steps + self._trains = trains + self._tasks = tasks + self._settings = settings + self._group_size = group_size + self._groups_per_batch = min(groups_per_batch, len(tasks)) + self._epochs = epochs + self._seed = seed + self._shuffle = shuffle + self._per_epoch = len(tasks) // self._groups_per_batch + + def get_batch(self, index: int) -> Sequence[EnvGroupBuilder]: + epoch, at = divmod(index, self._per_epoch) + order = list(range(len(self._tasks))) + if self._shuffle: + random.Random(self._seed + epoch).shuffle(order) + picked = order[at * self._groups_per_batch:(at + 1) * self._groups_per_batch] + group = RunGroup(self._run_name, self._steps, self._steps.trained(index) if self._trains else None) + return [EpisodeGroupBuilder(name, spec, reward, self._settings, self._group_size, group) + for name, spec, reward in (self._tasks[i] for i in picked)] + + def __len__(self) -> int: + return self._per_epoch * self._epochs + + +@chz.chz +class AgentEnvDatasetBuilder(RLDatasetBuilder): + """The tasks of the bundle folder ``bundle`` for ``rl.train``, prepared as ``agent-env run`` prepares them, their + episodes filed in run groups named ``/train|eval/step-``. ``train_split`` and ``test_split`` are + ``all`` or an eval of the bundle. ``reward`` is the step id whose score is the reward, needed + when a task records more than one. ``sandbox`` is where episodes deploy, as ``agent-env run --sandbox`` sets it, + and ``max_concurrent_episodes`` bounds the runs under way at once. A rollout the cookbook ends itself is scored + as it is cut, and given ``failed_reward`` on top.""" + + bundle: str + run_name: str + model_name: str + renderer_name: str + group_size: int + groups_per_batch: int + train_split: str = "all" + test_split: str | None = None + test_group_size: int = 1 + reward: str | None = None + sandbox: str | None = None + max_turns: int = 30 + tool_timeout: float = 60.0 + turn_timeout: float | None = None + max_concurrent_episodes: int = 8 + max_trajectory_tokens: int | None = None + failed_reward: float = 0.0 + epochs: int = 1 + seed: int = 0 + + async def __call__(self) -> tuple[RLDataset, RLDataset | None]: + wanted = [split for split in (self.train_split, self.test_split) if split is not None] + splits = await asyncio.to_thread(_splits, Path(self.bundle).expanduser(), wanted, self.sandbox) + settings = _Settings(model_name=self.model_name, renderer_name=self.renderer_name, sandbox=self.sandbox, + max_turns=self.max_turns, tool_timeout=self.tool_timeout, + max_concurrent_episodes=self.max_concurrent_episodes, + max_trajectory_tokens=self.max_trajectory_tokens, failed_reward=self.failed_reward, + tag=Path(self.bundle).name) + steps = TrainingSteps() + train = EpisodeDataset(playable(splits[self.train_split], self.reward, self.turn_timeout), settings, + group_size=self.group_size, groups_per_batch=self.groups_per_batch, epochs=self.epochs, + seed=self.seed, shuffle=True, run_name=self.run_name, steps=steps, trains=True) + test = None + if self.test_split is not None: + tasks = playable(splits[self.test_split], self.reward, self.turn_timeout) + test = EpisodeDataset(tasks, settings, group_size=self.test_group_size, groups_per_batch=len(tasks), + epochs=1, seed=self.seed, shuffle=False, run_name=self.run_name, + steps=steps, trains=False) + return train, test + + +def _splits(bundle: Path, splits: list[str], sandbox: str | None) -> dict[str, list[tuple[str, Task]]]: + """Each split's tasks with their names, ``all`` or an eval's, prepared as ``agent-env run`` prepares them before + its first task, without the agent the trainer plays (``played``), as the OpenEnv server prepares a bundle.""" + with namespace_routing(): + every = [entry.name for entry in parse_bundle(bundle).entries if entry.kind is BundleKind.TASK] + prepared = prepare_bundle(bundle, tasks=every, evals=[split for split in splits if split != "all"], + sandbox=sandbox, on_progress=logger.info, played=True) + tasks = [(entry.name, task) for entry, task in prepared.tasks] + by_id = {task.id: (name, task) for name, task in tasks} + found = {"all": tasks, **{entry.entry.name: [by_id[ref.id] for ref in entry.references] + for entry in prepared.plan.evals}} + return {split: found[split] for split in splits} + + +def _gives_skills(step: Any, agents: Sequence[str]) -> bool: + return getattr(step, "agent_name", None) in agents and ( + step.type == "add_skills" or step.type == "deploy_agent" and bool(step.skills)) + + +def playable(tasks: list[tuple[str, Task]], reward: str | None, + turn_timeout: float | None) -> list[tuple[str, EpisodeSpec, Reward]]: + """Each task that can run as an episode, with its reward; the rest are logged and left out. Raises ValueError when + a task's scores make no reward, since a rollout must be rewarded, or when no task is left. + + A task that gives a played agent skills is left out, here rather than when its turn opens: no trainer shows its + model skills yet, and a rollout refused at that turn would be retried and refused again.""" + episodes, refused, unscorable = [], {}, {} + for name, task in tasks: + try: + spec = EpisodeSpec.from_task(task, turn_timeout=turn_timeout) + except EpisodeError as e: + refused[name] = str(e) + continue + if skilled := [step.id for step in task.steps if _gives_skills(step, spec.trainer_agents)]: + refused[name] = (f"task {task.id!r} gives the agent it plays skills ({skilled}), which no trainer shows " + "its model yet") + continue + task_reward, problem = reward_of(task, reward) + if problem or not any(getattr(step, "verifier_id", None) for step in task.steps): + unscorable[name] = problem or f"task {task.id!r} records no score" + continue + episodes.append((name, spec, task_reward)) + for why in refused.values(): + logger.warning("left out: %s", why) + if unscorable: + raise ValueError("reward= can't make a reward of every task's scores:\n" + "\n".join(unscorable.values())) + if not episodes: + raise ValueError(f"no task can be played as an episode: {refused}") + return episodes diff --git a/tst/tinker/__init__.py b/tst/tinker/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tst/tinker/conftest.py b/tst/tinker/conftest.py new file mode 100644 index 000000000..5cd349748 --- /dev/null +++ b/tst/tinker/conftest.py @@ -0,0 +1,6 @@ +"""The Episode API's fake steps, stand-in teardown and tool sessions, and the local stores, which these tests play +through the Tinker helper's episodes.""" + +from tst.unit.conftest import local_stores # noqa: F401 +from tst.unit.openenv.environment_test import tools # noqa: F401 +from tst.unit.task.episode.run_test import teardowns # noqa: F401 diff --git a/tst/tinker/episode_env_test.py b/tst/tinker/episode_env_test.py new file mode 100644 index 000000000..5b9c701c4 --- /dev/null +++ b/tst/tinker/episode_env_test.py @@ -0,0 +1,260 @@ +"""The Tinker helper's episodes on the Episode API's fake steps, as tinker-cookbook's rollouts step them: a rollout's +tool calls run on its turn and its reply ends it with the task's reward, filed in its run group; a later turn's new +system prompt comes before its prompt; a turn with skills is refused; the turn bound ends a turn; a step's logs are +clipped; a rollout whose first prompt can't be written is abandoned and frees its slot; a rollout the cookbook cuts is +scored as it is cut, freeing its slot for the rest of its group; a bundle's splits as prepared for a trainer to play; +and the tasks a dataset plays.""" + +import asyncio +import json +import logging +from types import SimpleNamespace + +import pytest + +from agent_env.openenv.reward import _score_at +from agent_env.runner import store as run_store +from agent_env.task import EpisodeError, Task +from tst.unit.bundle._support import layout +from tst.unit.task.episode.run_test import ALICE, ITEMS, REPLIES, TURN, _spec +from tst.util.capabilities import TINKER, missing_capability_reason + +tinker = pytest.importorskip("agent_env.training.tinker", reason=missing_capability_reason(TINKER), + exc_type=ImportError) +base = pytest.importorskip("tinker_cookbook.renderers.base", reason=missing_capability_reason(TINKER)) +sdk = pytest.importorskip("tinker", reason=missing_capability_reason(TINKER)) + +GROUP = "dock-run/train/step-0003" +ADD_ANOTHER = {**TURN, "id": "again", "prompt_id": "again", "prompt": "Add another.", "depends_on": ["turn"]} + + +class _Renderer: + """Stands in for a cookbook renderer: the system prompt and the names of the tools it is given, as one message, + and a conversation as a token per message.""" + + def create_conversation_prefix_with_tools(self, tools, system_prompt=""): + return [{"role": "system", "content": json.dumps({"system": system_prompt, + "tools": [tool["name"] for tool in tools]})}] + + def get_stop_sequences(self): + return [] + + def build_generation_prompt(self, messages, **kwargs): + return sdk.ModelInput.from_ints(list(range(len(messages)))) + + +def _env(*steps, max_turns=8): + return tinker.EpisodeEnv("dock", _spec(*steps), renderer=_Renderer(), reward=_score_at("replies"), + model_name="policy", sandbox=None, max_turns=max_turns, tool_timeout=5, + max_concurrent_episodes=4, run_group_id=GROUP) + + +def _call(name, **arguments): + return base.ToolCall(function=base.ToolCall.FunctionBody(name=name, arguments=json.dumps(arguments)), + id=f"call-{name}") + + +@pytest.mark.asyncio +async def test_a_rollouts_calls_run_on_its_turn_and_its_reply_ends_it_with_the_tasks_reward_in_its_group( + teardowns, tools): + env = _env(ITEMS, ALICE, TURN, REPLIES) + + system, user = await env.initial_observation() + assert json.loads(system["content"]) == {"system": "You keep the items.", "tools": ["add_item"]} + assert user == {"role": "user", "content": "Add a crate."} + called = await env.step({"role": "assistant", "content": "", "tool_calls": [_call("add_item", item="crate")]}) + assert not called.episode_done + assert called.next_messages[-1] == {"role": "tool", "content": "add_item {'item': 'crate'}", + "tool_call_id": "call-add_item", "name": "add_item"} + ended = await env.step({"role": "assistant", "content": "Added a crate."}) + + assert (ended.episode_done, ended.reward) == (True, 1.0) + assert {key: ended.metrics[key] for key in ("score/replies", "messages", "tool_calls", "tool_errors")} == { + "score/replies": 1.0, "messages": 2, "tool_calls": 1, "tool_errors": 0} + run = run_store.get_run(f"episode-{ended.logs['episode']}") + assert (run.status, run.overrides["metadata"]["run_group_id"]) == ("COMPLETED", GROUP) + assert await env.score_cut() == (0.0, {"cut": 0.0}) + + +@pytest.mark.asyncio +async def test_a_later_turns_new_system_prompt_comes_before_its_prompt_as_every_trainer_writes_it(teardowns, tools): + env = _env(ITEMS, ALICE, TURN, {**ADD_ANOTHER, "system_prompt": "You count the items."}, REPLIES) + await env.initial_observation() + + second = await env.step({"role": "assistant", "content": "Added a crate."}) + assert (second.episode_done, second.next_messages[-1]) == ( + False, {"role": "user", "content": "You count the items.\n\nAdd another."}) + ended = await env.step({"role": "assistant", "content": "Added another."}) + + assert ended.episode_done + + +@pytest.mark.asyncio +async def test_a_turn_that_gives_the_agent_skills_is_refused_whichever_turn_it_is(teardowns, tools): + env = _env(ITEMS, ALICE, TURN, REPLIES) + await env.initial_observation() + later = SimpleNamespace(step_id="again", skills=[SimpleNamespace(name="refunds")]) + + with pytest.raises(EpisodeError, match=r"turn 'again' gives the agent skills \['refunds'\]"): + await env._open(later) + await env.close() + + +@pytest.mark.asyncio +async def test_a_turn_ends_at_the_bound_on_the_models_messages_with_no_reply(teardowns, tools): + env = _env(ITEMS, ALICE, TURN, REPLIES, max_turns=1) + await env.initial_observation() + + ended = await env.step({"role": "assistant", "content": "", "tool_calls": [_call("add_item", item="crate")]}) + + assert (ended.episode_done, ended.reward, ended.metrics["tool_calls"]) == (True, 0.0, 1) + + +@pytest.mark.asyncio +async def test_a_steps_logs_clip_the_models_message_and_each_call(teardowns, tools, monkeypatch): + monkeypatch.setattr(tinker.EpisodeEnv, "LOGGED_CHARS", 8) + env = _env(ITEMS, ALICE, TURN, REPLIES) + await env.initial_observation() + + called = await env.step({"role": "assistant", "content": "Adding a crate now.", + "tool_calls": [_call("add_item", item="crate")]}) + await env.close() + + assert called.logs["assistant"] == "Adding a… (11 more characters)" + assert called.logs["tool_call_0"] == ('add_item({"item":… (9 more characters)) -> ' + 'add_item… (18 more characters)') + + +@pytest.mark.asyncio +async def test_a_rollout_the_cookbook_cut_has_its_open_turn_ended_and_is_scored_once(teardowns, tools): + env = _env(ITEMS, ALICE, TURN, REPLIES) + await env.initial_observation() + + reward, metrics = await env.score_cut() + + assert (reward, metrics["score/replies"], metrics["cut"]) == (0.0, 0.0, 1.0) + assert await env.score_cut() == (0.0, {"cut": 0.0}) + + +class _ToollessRenderer(_Renderer): + """Stands in for a cookbook renderer that can't write tools, as ``role_colon`` can't.""" + + def create_conversation_prefix_with_tools(self, tools, system_prompt=""): + raise NotImplementedError("this renderer does not support tool calling") + + +@pytest.mark.asyncio +async def test_a_rollout_whose_first_prompt_cant_be_written_is_abandoned_and_frees_its_slot(teardowns, tools): + env = tinker.EpisodeEnv("dock", _spec(ITEMS, ALICE, TURN, REPLIES), renderer=_ToollessRenderer(), + reward=_score_at("replies"), model_name="policy", sandbox=None, max_turns=8, + tool_timeout=5, max_concurrent_episodes=1, run_group_id=GROUP) + + with pytest.raises(NotImplementedError, match="does not support tool calling"): + await env.initial_observation() + + assert not tinker._slots(1).locked() + assert run_store.get_run(f"episode-{env._episode.id}").status == "CANCELED" + + +def _group(group_size, max_concurrent_episodes, max_trajectory_tokens=None): + settings = tinker._Settings(model_name="policy", renderer_name="stub", sandbox=None, max_turns=8, tool_timeout=5, + max_concurrent_episodes=max_concurrent_episodes, + max_trajectory_tokens=max_trajectory_tokens, failed_reward=-0.5, tag="dock") + settings.renderer = _Renderer + steps = tinker.TrainingSteps() + return tinker.EpisodeGroupBuilder("dock", _spec(ITEMS, ALICE, TURN, REPLIES), _score_at("replies"), settings, + group_size, tinker.RunGroup("dock-run", steps, steps.trained(3))) + + +@pytest.mark.asyncio +async def test_rollouts_the_cookbook_cuts_free_their_slots_so_a_group_larger_than_the_slots_finishes( + teardowns, tools): + group = _group(group_size=3, max_concurrent_episodes=1) + envs = await group.make_envs() + + async def cut_at_the_token_limit(env): + await env.initial_observation() + return await env.step([], extra={"stop_reason": "length"}) + + try: + # Each rollout holds the one slot until it is scored: unscored when cut, the next rollout would never start. + cut = await asyncio.wait_for(asyncio.gather(*(cut_at_the_token_limit(env) for env in envs)), timeout=60) + finally: + await group.cleanup() + + assert [(step.episode_done, step.reward, step.metrics["cut"], step.metrics["score/replies"]) for step in cut] == [ + (True, -0.5, 1.0, 0.0)] * 3 + assert {run_store.get_run(f"episode-{env.message_env._episode.id}").status for env in envs} == {"COMPLETED"} + + +@pytest.mark.asyncio +async def test_a_rollout_whose_prompt_is_over_the_token_budget_is_scored_and_frees_its_slot(teardowns, tools): + group = _group(group_size=2, max_concurrent_episodes=1, max_trajectory_tokens=1) + envs = await group.make_envs() + + try: + overflows = await asyncio.wait_for(asyncio.gather(*(env.initial_observation() for env in envs)), timeout=60) + finally: + await group.cleanup() + + assert [(type(o).__name__, o.reward, o.metrics["cut"]) for o in overflows] == [ + ("InitialObservationOverflow", -0.5, 1.0)] * 2 + + +def test_a_split_is_an_evals_tasks_prepared_without_the_agent_the_trainer_plays(teardowns, tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + played = json.dumps([ITEMS, {**ALICE, "a2a_agent_id": "nobody", "env_ids": []}, TURN, REPLIES]) + root = layout(tmp_path / "dock", {"tasks/crate.json": played, "tasks/again.json": played, + "evals/train.toml": 'tasks = ["crate"]\n'}) + + splits = tinker._splits(root, ["all", "train"], None) + + assert {split: sorted(name for name, _ in tasks) for split, tasks in splits.items()} == { + "all": ["again", "crate"], "train": ["crate"]} + + +ENV = {"id": "env", "type": "deploy_env", "env_id": "retail"} +AGENT = {"id": "agent", "type": "deploy_agent", "agent_name": "policy", "env_ids": ["retail"], + "role": "agent@retail.example", "system_prompt": "You run the store."} +PLAY = {"id": "play", "type": "prompt_agent", "agent_name": "policy", "prompt_id": "play", "prompt": "Help."} +AGAIN = {**PLAY, "id": "again", "prompt_id": "again", "prompt": "Anything else?"} +REFUNDS = {"name": "refunds", "description": "How the store refunds.", "body": "Refund within 30 days."} +BOB = {"id": "bob", "type": "deploy_agent", "agent_name": "bob", "a2a_agent_id": "echo"} +ASK_BOB = {"id": "ask-bob", "type": "prompt_agent", "agent_name": "bob", "prompt": "Check."} + + +def _verifier(name): + return {"id": name, "type": "env_outcome_verifier", "env_id": "retail", "file_artifact_id": "check", + "verifier_id": name} + + +def _task(id, *steps): + return id, Task.from_dict({"id": id, "version": 1, "steps": [dict(step) for step in steps]}) + + +def test_tasks_that_cant_run_as_an_episode_are_left_out_and_the_rest_rewarded(caplog): + played = _task("played", ENV, AGENT, PLAY, _verifier("records"), _verifier("db_match")) + new_instructions = _task("new-instructions", ENV, AGENT, PLAY, {**AGAIN, "system_prompt": "You audit the store."}, + _verifier("records")) + two_agents = _task("two-agents", ENV, AGENT, PLAY, BOB, ASK_BOB, _verifier("records")) + skilled = _task("skilled", ENV, AGENT, PLAY, {"id": "learn", "type": "add_skills", "agent_name": "policy", + "skills": [REFUNDS]}, AGAIN, _verifier("records")) + skilled_at_deploy = _task("skilled-at-deploy", ENV, {**AGENT, "skills": [REFUNDS]}, PLAY, _verifier("records")) + with caplog.at_level(logging.WARNING): + kept = tinker.playable([played, new_instructions, two_agents, skilled, skilled_at_deploy], "records", None) + assert [(name, spec.trainer_agents) for name, spec, _ in kept] == [("played", ("policy",)), + ("new-instructions", ("policy",))] + assert kept[0][2]({"records": 0.25, "db_match": 0.0}) == 0.25 + assert "two-agents" in caplog.text + assert "'skilled' gives the agent it plays skills (['learn'])" in caplog.text + assert "'skilled-at-deploy' gives the agent it plays skills (['agent'])" in caplog.text + + +@pytest.mark.parametrize(("tasks", "reward", "refusal"), [ + ([_task("unscored", ENV, AGENT, PLAY)], None, "records no score"), + ([_task("two", ENV, AGENT, PLAY, _verifier("records"), _verifier("db_match"))], None, "name the one"), + ([_task("two-agents", ENV, AGENT, PLAY, BOB, ASK_BOB, _verifier("records"))], "records", "no task can be played"), +]) +def test_a_dataset_whose_tasks_make_no_reward_or_no_episode_is_refused(tasks, reward, refusal): + with pytest.raises(ValueError, match=refusal): + tinker.playable(tasks, reward, None) diff --git a/tst/unit/test_optional_extras_stay_optional.py b/tst/unit/test_optional_extras_stay_optional.py index 48b45cf6f..d23f1ec38 100644 --- a/tst/unit/test_optional_extras_stay_optional.py +++ b/tst/unit/test_optional_extras_stay_optional.py @@ -35,6 +35,7 @@ def find_spec(self, fullname, path=None, target=None): "agent_env.providers.sandbox_providers.sail_vm", "agent_env.providers.sandbox_providers.smol_vm", "agent_env.openenv", "agent_env.training.nemo_gym", "agent_env.training.verl", "agent_env.training.skyrl", "agent_env.training.harness", "agent_env.training.miles", "agent_env.training.slime", "agent_env.training._chat", + "agent_env.training.tinker", "agent_env.env.gateway", "agent_env.task_step.task_steps.run_code_runner", ) diff --git a/tst/unit/training/episodes_test.py b/tst/unit/training/episodes_test.py new file mode 100644 index 000000000..d73a46598 --- /dev/null +++ b/tst/unit/training/episodes_test.py @@ -0,0 +1,52 @@ +"""The arguments a rollout's tool calls run with, typed by each tool's schema, a later turn's prompt as every trainer's +model reads it, and the run groups a trainer files its episodes in by training step.""" + +from agent_env.training._episodes import RunGroup, TrainingSteps, next_prompt, run_group_id, typed_arguments + +LOOKUP = {"type": "object", "properties": { + "zip": {"type": "string"}, + "note": {"type": ["string", "null"]}, + "order": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "count": {"type": "integer"}, + "price": {"type": "number"}, + "gift": {"type": "boolean"}, + "items": {"type": "array", "items": {"type": "string"}}, +}} + +def test_a_scalar_its_schema_wants_as_a_string_is_given_back_as_the_string_it_was_written_as(): + assert typed_arguments({"zip": 19122, "note": 7, "order": True}, LOOKUP) == { + "zip": "19122", "note": "7", "order": "true"} + + +def test_values_their_schema_allows_are_left_as_parsed(): + arguments = {"zip": "19122", "note": None, "count": 3, "price": 2, "gift": False, "items": ["1151293680"]} + assert typed_arguments(arguments, LOOKUP) == arguments + + +def test_a_later_turns_new_system_prompt_comes_before_its_prompt_and_stays_in_effect(): + assert next_prompt("Add another.", "Keep the barrels.", "Keep the items.") == ( + "Keep the barrels.\n\nAdd another.", "Keep the barrels.") + assert next_prompt("Add another.", "Keep the items.", "Keep the items.") == ("Add another.", "Keep the items.") + assert next_prompt("Add another.", None, "Keep the items.") == ("Add another.", "Keep the items.") + assert next_prompt("Add another.", "", None) == ("Add another.", None) + + +def test_an_argument_the_schema_doesnt_name_and_arguments_that_arent_an_object_are_left_alone(): + assert typed_arguments({"other": 5}, LOOKUP) == {"other": 5} + assert typed_arguments([19122], LOOKUP) == [19122] + + +def test_a_group_is_filed_under_its_run_phase_and_step_and_an_evaluation_at_the_step_it_measures(): + steps = TrainingSteps() + assert steps.evaluated() == 0 + assert [steps.trained(index) for index in range(4)] == [0, 1, 2, 3] + assert steps.evaluated() == 4 + assert run_group_id("retail-run2", "eval", steps.evaluated()) == "retail-run2/eval/step-0004" + + +def test_an_evaluations_group_made_once_is_filed_at_the_step_each_time_it_is_played(): + steps = TrainingSteps() + evaluation = RunGroup("retail-run2", steps) + assert evaluation.id() == "retail-run2/eval/step-0000" + trained = RunGroup("retail-run2", steps, steps.trained(0)) + assert (trained.id(), evaluation.id()) == ("retail-run2/train/step-0000", "retail-run2/eval/step-0001") diff --git a/tst/util/capabilities.py b/tst/util/capabilities.py index 1d4d46d17..df1a38244 100644 --- a/tst/util/capabilities.py +++ b/tst/util/capabilities.py @@ -63,6 +63,10 @@ #: SGLang and Megatron-LM and CPU torch aren't in the dev extra, and only that tier's CI job installs them. SLIME = "slime_installed" +#: The Tinker tier (``tst/tinker``) plays episodes through ``agent_env.training.tinker`` on tinker-cookbook itself; +#: tinker-cookbook and torch aren't in the dev extra, and only that tier's CI job installs them. +TINKER = "tinker_installed" + #: ``collect_artifacts`` sizes a file with ``stat -c %s``, which only GNU stat takes (macOS ships BSD stat), #: so collecting off a local sandbox needs GNU stat first on PATH. GNU_STAT = "gnu_stat" diff --git a/uv.lock b/uv.lock index c38970fe0..e3534201e 100644 --- a/uv.lock +++ b/uv.lock @@ -180,6 +180,9 @@ tenki = [ tensorlake = [ { name = "tensorlake" }, ] +tinker = [ + { name = "openenv" }, +] trl = [ { name = "datasets" }, { name = "openenv" }, @@ -201,12 +204,13 @@ verl = [ [package.metadata] requires-dist = [ { name = "a2a-sdk", specifier = "==0.3.26" }, - { name = "agentenv-framework", extras = ["aws", "boat", "boxd", "coreweave", "daytona", "gcp", "harness", "openenv", "sail", "skyrl", "tensorlake", "tenki", "trl", "vercel", "verl"], marker = "extra == 'dev'" }, + { name = "agentenv-framework", extras = ["aws", "boat", "boxd", "coreweave", "daytona", "gcp", "harness", "openenv", "sail", "skyrl", "tensorlake", "tenki", "tinker", "trl", "vercel", "verl"], marker = "extra == 'dev'" }, { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'harness'" }, { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'miles'" }, { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'nemo-gym'" }, { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'skyrl'" }, { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'slime'" }, + { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'tinker'" }, { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'trl'" }, { name = "agentenv-framework", extras = ["openenv"], marker = "extra == 'verl'" }, { name = "agentenv-framework-protocol", extras = ["computer"], editable = "packages/agentenv-protocol" }, @@ -277,7 +281,7 @@ requires-dist = [ { name = "vercel-sandbox", marker = "extra == 'vercel'", specifier = ">=0.7.0,<0.8" }, { name = "websockets", marker = "extra == 'dev'", specifier = ">=13" }, ] -provides-extras = ["openenv", "harness", "trl", "nemo-gym", "verl", "skyrl", "miles", "slime", "aws", "gcp", "sail", "vercel", "tensorlake", "tenki", "boxd", "daytona", "boat", "coreweave", "dev", "smol"] +provides-extras = ["openenv", "harness", "trl", "nemo-gym", "verl", "skyrl", "miles", "slime", "tinker", "aws", "gcp", "sail", "vercel", "tensorlake", "tenki", "boxd", "daytona", "boat", "coreweave", "dev", "smol"] [[package]] name = "agentenv-framework-protocol" From e6e52436c3e23a3ce4462388b05dfabe05d501cb Mon Sep 17 00:00:00 2001 From: Tejas Polakam Date: Tue, 17 Nov 2026 11:33:34 -0500 Subject: [PATCH 5/5] fix(training): play every task once a pass, its last batch the tasks left over EpisodeDataset rounded a pass's batches down, so with 10 tasks and groups_per_batch=3 one task went unplayed each pass. A pass now rounds up, its last batch the tasks left over, as the cookbook's own RL datasets end a pass. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/training/tinker.py | 9 ++++++--- tst/tinker/episode_env_test.py | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/agent_env/training/tinker.py b/src/agent_env/training/tinker.py index 0f36e59ed..dc8567932 100644 --- a/src/agent_env/training/tinker.py +++ b/src/agent_env/training/tinker.py @@ -27,6 +27,7 @@ import dataclasses import json import logging +import math import random from collections.abc import Callable, Sequence from pathlib import Path @@ -373,8 +374,10 @@ def renderer(self) -> Renderer: class EpisodeDataset(RLDataset): - """A split's tasks in batches of ``groups_per_batch`` groups, shuffled anew each of ``epochs`` passes; a group's - episodes are filed under the run, whether they train or evaluate, and the training step that plays them.""" + """A split's tasks in batches of ``groups_per_batch`` groups, shuffled anew each of ``epochs`` passes, each pass + playing every task once: its last batch is the tasks left over, as the cookbook's own RL datasets end a pass. A + group's episodes are filed under the run, whether they train or evaluate, and the training step that plays + them.""" def __init__(self, tasks: list[tuple[str, EpisodeSpec, Reward]], settings: _Settings, *, group_size: int, groups_per_batch: int, epochs: int, seed: int, shuffle: bool, run_name: str, @@ -389,7 +392,7 @@ def __init__(self, tasks: list[tuple[str, EpisodeSpec, Reward]], settings: _Sett self._epochs = epochs self._seed = seed self._shuffle = shuffle - self._per_epoch = len(tasks) // self._groups_per_batch + self._per_epoch = math.ceil(len(tasks) / self._groups_per_batch) def get_batch(self, index: int) -> Sequence[EnvGroupBuilder]: epoch, at = divmod(index, self._per_epoch) diff --git a/tst/tinker/episode_env_test.py b/tst/tinker/episode_env_test.py index 5b9c701c4..a4a54e9a1 100644 --- a/tst/tinker/episode_env_test.py +++ b/tst/tinker/episode_env_test.py @@ -3,7 +3,7 @@ system prompt comes before its prompt; a turn with skills is refused; the turn bound ends a turn; a step's logs are clipped; a rollout whose first prompt can't be written is abandoned and frees its slot; a rollout the cookbook cuts is scored as it is cut, freeing its slot for the rest of its group; a bundle's splits as prepared for a trainer to play; -and the tasks a dataset plays.""" +the tasks a dataset plays; and its batches, each pass playing every task once.""" import asyncio import json @@ -258,3 +258,19 @@ def test_tasks_that_cant_run_as_an_episode_are_left_out_and_the_rest_rewarded(ca def test_a_dataset_whose_tasks_make_no_reward_or_no_episode_is_refused(tasks, reward, refusal): with pytest.raises(ValueError, match=refusal): tinker.playable(tasks, reward, None) + + +def test_each_pass_plays_every_task_once_its_last_batch_the_tasks_left_over(): + tasks = tinker.playable([_task(f"task-{n}", ENV, AGENT, PLAY, _verifier("records")) for n in range(10)], + "records", None) + settings = tinker._Settings(model_name="policy", renderer_name="stand-in", sandbox=None, max_turns=8, + tool_timeout=5, max_concurrent_episodes=4, max_trajectory_tokens=None, + failed_reward=0.0, tag="dock") + dataset = tinker.EpisodeDataset(tasks, settings, group_size=2, groups_per_batch=3, epochs=2, seed=0, shuffle=True, + run_name="dock-run", steps=tinker.TrainingSteps(), trains=True) + + batches = [[group.name for group in dataset.get_batch(index)] for index in range(len(dataset))] + + assert [len(batch) for batch in batches] == [3, 3, 3, 1] * 2 + for one_pass in (batches[:4], batches[4:]): + assert sorted(name for batch in one_pass for name in batch) == sorted(name for name, _, _ in tasks)