diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cb017e5b..d20e4cbf1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,65 @@ jobs: - name: Run tests run: uv run --python ${{ matrix.python-version }} --with=".[dev]" pytest --cov --cov-report='' + templates: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + template: [blank, coding, cua] + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Create example environment + run: uv run hud init "${{ runner.temp }}/${{ matrix.template }}" --template "${{ matrix.template }}" + + - name: Install example with the SDK from this checkout + working-directory: ${{ runner.temp }}/${{ matrix.template }} + run: | + uv sync --all-extras + uv pip install -e "$GITHUB_WORKSPACE" + + - name: Run example checks + working-directory: ${{ runner.temp }}/${{ matrix.template }} + run: | + uv run --no-sync hud task list --source . + if [ -d tests ]; then + uv run --no-sync ruff format . --check + uv run --no-sync ruff check . + uv run --no-sync pytest -q + fi + + - name: Build example image + working-directory: ${{ runner.temp }}/${{ matrix.template }} + run: docker build --file Dockerfile.hud --tag "hud-example:${{ matrix.template }}" . + + - name: Start example image + run: | + docker run --detach --name hud-example --publish 8765:8765 --shm-size 2g \ + "hud-example:${{ matrix.template }}" + + - name: Inspect example image + working-directory: ${{ runner.temp }}/${{ matrix.template }} + run: | + for attempt in {1..60}; do + if (echo > /dev/tcp/127.0.0.1/8765) 2>/dev/null; then + uv run --no-sync hud client info --url tcp://127.0.0.1:8765 + exit 0 + fi + sleep 1 + done + docker logs hud-example + exit 1 + + - name: Stop example image + if: always() + run: docker rm --force hud-example || true + lint-ruff: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 409d4254b..5426ec1ea 100644 --- a/.gitignore +++ b/.gitignore @@ -62,7 +62,9 @@ hud/rl/checkpoints_test/ docs/internal -environments/ +environments/* +!environments/coding/ +!environments/cua/ experiments/ .memories/ diff --git a/README.md b/README.md index 4a06edc15..e44c744d8 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,12 @@ hud set HUD_API_KEY=your-key-here # or: export HUD_API_KEY=your-key-here ``` -Then scaffold your first environment: +Then start from the coding environment, or choose another example environment: ```bash hud init my-env +hud init my-desktop-env --template cua +hud init my-custom-env --template blank ``` ![Agent running on SheetBench](docs/src/images/trace_sheet.gif) @@ -85,8 +87,8 @@ For local iteration, the same protocol works against a container on your laptop: ```bash docker build -f Dockerfile.hud -t my-env . docker run -d --name run1 -p 8765:8765 my-env -hud task start fix_bug --url tcp://127.0.0.1:8765 -hud task grade fix_bug --url tcp://127.0.0.1:8765 --answer "..." +hud task start fix_bug --source tasks.py --url tcp://127.0.0.1:8765 +hud task grade fix_bug --source tasks.py --url tcp://127.0.0.1:8765 --answer "..." docker rm -f run1 ``` diff --git a/cookbooks/codex-coding/README.md b/cookbooks/codex-coding/README.md deleted file mode 100644 index e0ad4a9aa..000000000 --- a/cookbooks/codex-coding/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Codex Coding Agent - -Build your own [Codex](https://github.com/openai/codex) with the HUD SDK: an -environment exposes an `ssh` capability backed by a `Workspace`, and -`OpenAIAgent` drives it with OpenAI's native `shell` and `apply_patch` tools โ€” -the same protocol the `codex` CLI uses. - -## Run - -From this directory (requires `HUD_API_KEY` for gateway inference): - -```bash -uv run codex_agent.py - -# Custom task -uv run codex_agent.py --task "Create a Python script that prints the Fibonacci sequence" - -# Custom working directory -uv run codex_agent.py --work-dir ./codex_output -``` - -To run the same environment as a packaged, sandboxed box instead of on your -machine, see `hud deploy` and `RemoteSandbox` in the deploy docs. diff --git a/cookbooks/codex-coding/codex_agent.py b/cookbooks/codex-coding/codex_agent.py deleted file mode 100644 index ce489cb94..000000000 --- a/cookbooks/codex-coding/codex_agent.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python3 -""" -Build Your Own Codex - A Recreation of OpenAI's Codex CLI - -This cookbook shows how to build your own Codex (https://github.com/openai/codex) -from scratch using the HUD SDK. The environment runs a ``Workspace`` serving an -``ssh`` capability; the ``OpenAIAgent`` drives it with OpenAI's native -``shell`` and ``apply_patch`` tools โ€” the same protocol the ``codex`` CLI uses. - -What you get: -- **Your own Codex** - Same behavior as `codex` CLI, but fully customizable -- **Full observability** - Every tool call and response traced on hud.ai - -See the README in this directory for setup and usage. Requires ``HUD_API_KEY`` -(gateway inference). -""" - -import argparse -import asyncio -import os - -from dotenv import load_dotenv -from openai import AsyncOpenAI - -# Load .env file from current directory or parent directories -load_dotenv() - -import hud -from hud import LocalRuntime -from hud.agents.openai import OpenAIAgent -from hud.agents.types import OpenAIConfig -from hud.settings import settings - -# Codex-capable models that support native shell/apply_patch tools -CODEX_MODELS = { - "gpt-5.1-codex", - "gpt-5.1", - "gpt-5.3-codex", - "gpt-5.4", - "gpt-5.6", -} - -PROMPT_TEMPLATE = """You are a skilled software developer. Complete the following task: - -{task_description} - -Use the available tools: -- `shell` to run commands (ls, cat, python, etc.) -- `apply_patch` to create or modify files - -Work in the current directory. When done, verify your work runs correctly.""" - -# The environment this file *is*: `LocalRuntime(__file__)` serves it in a child -# process (which re-imports this module), so the task's prompt and grade -# arrive over the wire while the agent loop runs here. The workspace root is -# handed to that child via CODEX_WORK_DIR. Attaching the workspace writes -# nothing: the serving child starts it (SSH keys + socket) and publishes the -# shell capability when the env comes up. -WORK_DIR = os.path.abspath(os.environ.get("CODEX_WORK_DIR") or os.getcwd()) -env = hud.Environment("local-codex") -env.workspace(WORK_DIR) - - -@env.template() -async def coding_task(task_description: str): - yield PROMPT_TEMPLATE.format(task_description=task_description) - yield 1.0 # simple success - task completed - - -async def run_coding_task( - task: str, - model: str = "gpt-5.3-codex", - max_steps: int = 20, - work_dir: str | None = None, -) -> None: - """Run a coding task locally. - - The environment runs a ``Workspace`` on your machine serving an ``ssh`` - capability; the agent's shell commands and patches land in that directory. - """ - if model not in CODEX_MODELS: - raise ValueError( - f"Model '{model}' is not in the Codex-capable list {sorted(CODEX_MODELS)}.\n" - "Use a model that supports native shell/apply_patch tools." - ) - if not settings.api_key: - raise ValueError( - "HUD_API_KEY is required.\n" - "Get yours at: https://hud.ai/project/api-keys\n" - "Then: export HUD_API_KEY='sk-hud-...'" - ) - - base_path = os.path.abspath(work_dir) if work_dir else os.getcwd() - if not os.path.exists(base_path): - raise ValueError(f"Directory not found: {base_path}") - os.environ["CODEX_WORK_DIR"] = base_path # inherited by the spawned env process - - print(f"๐Ÿ“ Working directory: {base_path}") - - # Codex-capable OpenAIAgent routed through the HUD gateway. - model_client = AsyncOpenAI( - base_url=settings.hud_gateway_url, - api_key=settings.api_key, - ) - agent = OpenAIAgent(OpenAIConfig(model=model, model_client=model_client, max_steps=max_steps)) - - print("๐ŸŒ Using HUD Gateway for inference") - print(f"๐Ÿค– Model: {model}") - print(f"๐Ÿ“‹ Task: {task}") - print("=" * 60) - - job = await coding_task(task_description=task).run(agent, runtime=LocalRuntime(__file__)) - - print("=" * 60) - (run,) = job.runs - if run.trace.is_error: - print(f"โŒ Task failed: {run.trace.content}") - return - print("โœ… Task completed!") - print(f"๐Ÿ“Š Reward: {job.reward}") - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Run coding tasks with OpenAI's native shell and apply_patch tools", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - uv run codex_agent.py - - # Custom working directory - uv run codex_agent.py --work-dir ./codex_output - - # Custom task - uv run codex_agent.py \\ - --task "Create a Python script that prints the Fibonacci sequence up to 10 numbers" - - # Use a different Codex model - uv run codex_agent.py --model gpt-5.1-codex -""", - ) - parser.add_argument( - "--task", - type=str, - default="Create a Python script called main.py that prints 'Hello, World!' and the current date/time", - help="The coding task to complete", - ) - parser.add_argument( - "--model", - type=str, - default="gpt-5.3-codex", - help="Codex-capable OpenAI model (default: gpt-5.3-codex)", - ) - parser.add_argument( - "--max-steps", - type=int, - default=20, - help="Maximum agent steps (default: 20)", - ) - parser.add_argument( - "--work-dir", - type=str, - default=None, - help="Working directory for file operations (default: current directory)", - ) - return parser.parse_args() - - -async def main() -> None: - args = _parse_args() - await run_coding_task( - task=args.task, - model=args.model, - max_steps=args.max_steps, - work_dir=args.work_dir, - ) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/cookbooks/codex-coding/pyproject.toml b/cookbooks/codex-coding/pyproject.toml deleted file mode 100644 index 8be7f342d..000000000 --- a/cookbooks/codex-coding/pyproject.toml +++ /dev/null @@ -1,17 +0,0 @@ -[project] -name = "codex-coding" -version = "0.1.0" -description = "Build your own Codex with the HUD SDK (cookbook)" -requires-python = ">=3.11,<3.13" -dependencies = [ - "hud", - "python-dotenv", -] - -[tool.uv] -package = false - -# Track the SDK from this repo. If you copied this folder out, delete this -# block to use the released hud from PyPI. -[tool.uv.sources] -hud = { path = "../..", editable = true } diff --git a/docs/v6/cookbooks/coding-agent.mdx b/docs/v6/cookbooks/coding-agent.mdx index 17c9f0c91..6a27a479e 100644 --- a/docs/v6/cookbooks/coding-agent.mdx +++ b/docs/v6/cookbooks/coding-agent.mdx @@ -1,104 +1,117 @@ --- -title: "Coding agent" -description: "Run a coding agent inside a HUD shell-and-files environment, with tests as the grader, source code in the SDK repo, and a step-by-step walkthrough." +title: "Coding environment" +description: "Start from HUD's coding environment, run a real SWE-bench task, and adapt its isolated workspace and hidden-test grading to your repository." icon: "code" --- -A coding task gives the agent a shell and some files, asks for a change, and grades the result by running it. This example does exactly that: an environment with a sandboxed shell, a task that asks the agent to make a failing test pass, and a `BashGrader` that scores by running the test suite. +HUD's coding example is a complete repository environment: the agent works in a sandboxed `ssh` +workspace while the environment keeps the original Git history and grading logs outside that +workspace, then applies the authored tests after the agent finishes. `hud init --template coding` +copies the project as a starting point. -## The environment +## Create the environment -The agent works in a **workspace**: a sandboxed shell and file system it reaches over the `ssh` capability, which the env publishes when it serves. In `@env.initialize` we seed a buggy module and a test, then declare the task. Its grader runs `pytest` and scores by exit code. - -The grader runs an **authoritative copy of the test that lives outside the agent's workspace**. The agent gets its own copy to read and run, but if the grader re-ran that editable copy, the cheapest path to a passing `pytest` would be to weaken or delete the test - classic reward hacking. Keeping the scoring copy out of reach closes that path. - -```python env.py -from pathlib import Path - -from hud.environment import Environment -from hud.graders import BashGrader - -ROOT = Path("workspace").resolve() # the agent's directory -CHECKS = Path("checks").resolve() # grader-only, outside the workspace - -TEST = "from calc import add\n\ndef test_add():\n assert add(2, 3) == 5\n" +```bash +hud init my-coding-env --template coding +cd my-coding-env +uv sync +``` -env = Environment(name="coder") -env.workspace(ROOT) +The generated project contains: -@env.initialize -async def _seed(): - (ROOT / "calc.py").write_text("def add(a, b):\n return a - b\n") # bug - (ROOT / "test_calc.py").write_text(TEST) # the agent's copy - CHECKS.mkdir(exist_ok=True) - (CHECKS / "test_calc.py").write_text(TEST) # the authoritative copy +| Path | Purpose | +|------|---------| +| `env.py` | Repository setup, workspace lifecycle, and the `coding-task` template. | +| `tasks.py` | Concrete task rows, test patches, commands, and expected results. | +| `coding/repo.py` | Trusted Git setup, history isolation, diff capture, and restoration. | +| `coding/grading.py` | JUnit parsing and fail-to-pass/pass-to-pass scoring. | +| `flask.bundle` | Prepared repository baselines and reference fixes for the example tasks. | +| `Dockerfile.hud` | Reproducible dependencies and the packaged repository workspace. | -@env.template() -async def fix_add(target: str = "test_calc.py"): - yield f"There's a failing test in {target} in your workspace. Find and fix the bug so the test passes." - result = await BashGrader.grade( - weight=1.0, - command=f"python -m pytest {CHECKS / target} -q", - cwd=str(ROOT), - ) - yield result.value +The included `flask-4992` and `flask-5063` tasks are from +[SWE-bench Lite](https://huggingface.co/datasets/princeton-nlp/SWE-bench_Lite). Run them with any +coding agent that opens the environment's shell capability: -tasks = [fix_add()] +```bash +hud set HUD_API_KEY=your-key-here +hud eval tasks.py claude --task-ids flask-4992 flask-5063 --runtime local -y ``` -This task has no `answer = yield` - the deliverable is the **state of the workspace**, not a text answer. - - -To start from an existing repo instead of seeding files inline, write it into the workspace root in `@env.initialize`, or pass `mounts=` (see [Capabilities](/v6/reference/capabilities)). - - -## Run it - -Point a coding agent at the environment. `claude` opens the `ssh` capability, edits `calc.py`, and the grader re-runs the test: +Local non-root runs require usable Linux `bwrap` isolation. Otherwise, build the packaged environment +and attach to its control channel: ```bash -hud eval env.py claude +docker build -f Dockerfile.hud -t my-coding-env . +docker run -d --name my-coding-env -p 8765:8765 my-coding-env +hud eval tasks.py claude --task-ids flask-4992 flask-5063 --runtime tcp://127.0.0.1:8765 --max-concurrent 1 -y +docker rm -f my-coding-env ``` -For Claude Code (the `claude` CLI driving the shell over SSH), use the `ClaudeSDKAgent` in code: +## How the grading boundary works -```python run.py -import asyncio -from hud.agents import ClaudeSDKAgent -from hud.agents.types import ClaudeSDKConfig -from env import fix_add +1. Task setup checks out the baseline and moves the original Git history into the environment-owned + vault. +2. The agent receives a single-commit repository through the `ssh` workspace. The original history + and reference-fix refs are not reachable from that workspace. +3. Grading discards agent-controlled Git metadata, restores the trusted history, and captures only the + worktree diff against the setup snapshot. +4. The environment reapplies that diff, restores the paths changed by `test_patch` to the baseline, + applies the patch, and runs the authored `test_script`. +5. The JUnit report is scored against the configured fail-to-pass and pass-to-pass test IDs. -async def main(): - agent = ClaudeSDKAgent(ClaudeSDKConfig(model="claude-sonnet-4-5")) - job = await fix_add().run(agent) - print("reward:", job.reward) - -asyncio.run(main()) -``` +This keeps the deliverable as repository state rather than a text answer, while preventing the agent +from weakening the grader or reading the reference fix. -## Read the trace +## Define a task -Every step - the shell commands, the edit, the test run - is on the trace at [hud.ai](https://hud.ai). A reward of `1.0` means `pytest` exited `0`; `0.0` means the test still fails. +Call `coding_task` once per repository issue and give the resulting task a stable slug: -## Make it a dataset +```python tasks.py +from env import coding_task + +_fix_parser = coding_task( + description="Fix the parser without breaking existing inputs.", + test_script="python -m pytest -q {test_files} --junitxml={junit_path}", + test_patch="""diff --git a/test_parser.py b/test_parser.py +--- a/test_parser.py ++++ b/test_parser.py +@@ -1,3 +1,6 @@ + def test_existing_input(): + assert parse("old") ++ ++def test_new_input(): ++ assert parse("new") +""", + base_ref="origin/parser_baseline", + test_files=["test_parser.py"], + f2p_test_nodeids=["test_parser.TestParser.test_new_input"], + p2p_test_nodeids=["test_parser.TestParser.test_existing_input"], +) +_fix_parser.slug = "fix-parser" + +tasks = [_fix_parser] +``` -Parameterize the task definition and create concrete tasks for a spread of bugs: +`test_script` must write JUnit XML to `{junit_path}`. JUnit IDs use `classname.name`, not pytest's +slash-and-`::` collection syntax. Set `use_binary_score=True` when every selected test must pass; +otherwise the reward is the fraction of selected tests that pass. -```python tasks.py -from env import fix_add +## Adapt it to another repository -tasks = [fix_add(target=t) for t in ("test_calc.py", "test_utils.py", "test_io.py")] -``` +- Replace `flask.bundle` with a bundle containing the prepared baselines for your tasks. +- Update `REPO_SOURCE` in `env.py`, or set `REPO_URL` for a local run. +- Install repository dependencies in `Dockerfile.hud` so grading does not depend on runtime downloads. +- Keep reference fixes outside the baseline history exposed to the agent; put hidden test patches in + the task rows. +- Add task rows in `tasks.py`; changing rows does not require changing the environment template. - -`BashGrader` needs bash, so on native Windows it scores `0.0` - grade from macOS/Linux, WSL, or a built image. - +The complete source lives in +[`environments/coding`](https://github.com/hud-evals/hud-python/tree/main/environments/coding). ## See also - + - diff --git a/docs/v6/cookbooks/index.mdx b/docs/v6/cookbooks/index.mdx index ac0df8267..d1edca38a 100644 --- a/docs/v6/cookbooks/index.mdx +++ b/docs/v6/cookbooks/index.mdx @@ -4,19 +4,18 @@ description: "Complete, runnable cookbook examples you can copy and adapt: codin icon: "book-open" --- -Cookbooks are complete, runnable projects you can copy and adapt. Each one shows -an environment, an agent, and a grader working end to end, so you can see a whole -evaluation rather than one concept at a time. They all live in the -[`cookbooks/`](https://github.com/hud-evals/hud-python/tree/main/cookbooks) -directory of the SDK repo. The examples below with a **walkthrough** have a full -guide on this site; the rest are best read straight from their source. +The walkthroughs connect complete projects from the SDK repo to the concepts they demonstrate. +Environment projects live in [`environments/`](https://github.com/hud-evals/hud-python/tree/main/environments) +and can be copied with `hud init`; specialized agents, protocols, and training loops live in +[`cookbooks/`](https://github.com/hud-evals/hud-python/tree/main/cookbooks). Entries without a +walkthrough are best read from their source. ## Walkthroughs - - Run a coding agent against a shell + files environment, graded by tests. - Source: [`cookbooks/codex-coding`](https://github.com/hud-evals/hud-python/tree/main/cookbooks/codex-coding). + + Run a real SWE-bench task in an isolated repository workspace with hidden-test grading. + Source: [`environments/coding`](https://github.com/hud-evals/hud-python/tree/main/environments/coding). Serve a chat task over the A2A protocol and talk to it from any client. diff --git a/docs/v6/cookbooks/ops-diagnostics.mdx b/docs/v6/cookbooks/ops-diagnostics.mdx index 9f719bb10..b127f3e0f 100644 --- a/docs/v6/cookbooks/ops-diagnostics.mdx +++ b/docs/v6/cookbooks/ops-diagnostics.mdx @@ -84,6 +84,6 @@ Vary the incident to mint a dataset with a difficulty range - some with an obvio - + diff --git a/docs/v6/guides/creating-an-environment.mdx b/docs/v6/guides/creating-an-environment.mdx index 74020c525..d971cc6c5 100644 --- a/docs/v6/guides/creating-an-environment.mdx +++ b/docs/v6/guides/creating-an-environment.mdx @@ -13,10 +13,11 @@ On this page: [Start](#start) ยท [The `env.py` file](#the-envpy-file) ยท [Capabi ## Start -Run `hud init` in your terminal. This lets you select a template to start with, the default is a blank template. +Run `hud init` in your terminal. A named non-interactive run starts from the coding environment; an +interactive run lets you choose. This guide uses `blank` because it builds each part from scratch. ```bash -hud init my-env +hud init my-env --template blank cd my-env ``` @@ -306,5 +307,11 @@ A HUD environment isn't locked to HUD: its tasks export through the experimental ## See environment examples -For more to build on, pick a different starting point with `hud init`, or read the -[cookbooks](/v6/cookbooks/index) for worked environments like a coding agent and an ops diagnostics task. +Choose the example closest to the environment you are building: + +- `hud init my-env --template blank` - the minimal prompt-and-reward structure used in the quickstart. +- `hud init my-env --template coding` - an isolated repository workspace with hidden-test grading. +- `hud init my-env --template cua` - a supervised Linux desktop with deterministic and model-judged grading. + +The [cookbooks](/v6/cookbooks/index) build on these patterns for agents, training loops, protocols, +and domain-specific benchmarks. diff --git a/docs/v6/reference/capabilities.mdx b/docs/v6/reference/capabilities.mdx index 1413d3000..3efc968af 100644 --- a/docs/v6/reference/capabilities.mdx +++ b/docs/v6/reference/capabilities.mdx @@ -218,6 +218,12 @@ async def _down(): ``` + +For a complete desktop image with Xvfb, XFCE, Chromium, x11vnc, supervisor, tasks, and graders, run +`hud init my-desktop-env --template cua`. The reference implementation lives in +[`environments/cua`](https://github.com/hud-evals/hud-python/tree/main/environments/cua). + + ### `robot` - an observation/action loop ```text @@ -291,6 +297,7 @@ async def _down(): Use a relative path (`"workspace"`, created next to `env.py`). Sandbox isolation (`bwrap`) is Linux-only - unisolated elsewhere, isolated in a built image. +For a complete repository workspace with hidden-test grading, run `hud init my-coding-env --template coding`. | Parameter | Type | Description | diff --git a/docs/v6/reference/cli.mdx b/docs/v6/reference/cli.mdx index dd7659443..781f53e39 100644 --- a/docs/v6/reference/cli.mdx +++ b/docs/v6/reference/cli.mdx @@ -10,19 +10,24 @@ Install the CLI with `uv tool install hud --python 3.12`. Authenticate once with ### `hud init` -Scaffold a new environment package in a fresh `` directory. The `blank` template (the default with no preset, and the first option in the interactive picker) writes a minimal local scaffold - `env.py` (environment, templates, capabilities), `tasks.py` (concrete task rows), `Dockerfile.hud`, and `pyproject.toml` - no network, no API key. Every other preset downloads a starter environment from GitHub instead. +Start a new environment package in a fresh `` directory. HUD provides example environments from the SDK +repository; a named non-interactive run uses `coding` by default. In a source checkout, `hud init` +copies the matching `environments/` directory. An installed release +fetches the matching SDK tag and extracts that same directory, so the example always matches the +installed SDK rather than `main`. The `blank` example is the exception: it generates a minimal local +scaffold with no network access. ```bash -hud init # pick a template โ†’ ./