Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,16 @@ A **capability** is a connection the environment exposes; a **harness** attaches

From the [platform UI](https://hud.ai) you can run batches, compare models on the same taskset, and inspect every trace.

A **project** holds the environments and tasksets a team creates and decides who can see them. Deploying without one uses your team's default project, so nothing here is required to get started. To put an environment somewhere else, pin the directory once and every later `hud deploy` and `hud sync tasks` follows it:

```bash
hud project list # projects you can use
hud project use browser-evals # writes projectId to .hud/config.json
hud project # where does a deploy here land?
```

Both commands also take `--project <name-or-id>` to override the resolved Project. A successful deploy or sync pins that Project to the directory; `hud set HUD_PROJECT=<name-or-id>` instead sets a machine-wide default for directories you have not pinned. Precedence is the flag, then the directory's `.hud/config.json`, then `HUD_PROJECT`, then your team default. An environment or taskset that already exists stays where it is; naming a different project fails rather than moving it.

→ [Run & deploy](https://docs.hud.ai/v6/reference/runtime)

## Train on rewards
Expand Down
2 changes: 2 additions & 0 deletions hud/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
from .jobs import jobs_app # noqa: E402
from .login import login_command # noqa: E402
from .models import models_app # noqa: E402
from .project import project_app # noqa: E402
from .qa import qa_app # noqa: E402
from .serve import serve_command # noqa: E402
from .sync import sync_app # noqa: E402
Expand All @@ -54,6 +55,7 @@
app.add_typer(jobs_app, name="jobs")
app.add_typer(trace_app, name="trace")
app.add_typer(qa_app, name="qa")
app.add_typer(project_app, name="project")


@app.command(name="set")
Expand Down
23 changes: 22 additions & 1 deletion hud/cli/deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from hud.cli.utils.build_logs import poll_build_status, stream_build_logs
from hud.cli.utils.config import parse_env_file, parse_key_value
from hud.cli.utils.context import create_build_context_tarball, format_size
from hud.cli.utils.project import Placement, resolve_writable_placement
from hud.cli.utils.registry import get_registry_environment
from hud.cli.utils.source import EnvironmentSource
from hud.eval.runtime import RuntimeConfig
Expand All @@ -41,6 +42,7 @@
class _DeployPlan:
name: str
registry_id: str | None
placement: Placement
runtime: str | None
runtime_config: RuntimeConfig | None
env_vars: dict[str, str]
Expand Down Expand Up @@ -371,6 +373,7 @@ def _prepare_deploy_plan(
env_file: str | None,
no_env: bool,
registry_id: str | None,
project: str | None,
build_args: list[str] | None,
build_secrets: list[str] | None,
runtime: str | None,
Expand All @@ -386,6 +389,7 @@ def _prepare_deploy_plan(
platform,
console,
)
placement = resolve_writable_placement(platform, env_source, flag=project, console=console)
skip_dotenv = _skip_dotenv(
env_source,
env_dir,
Expand Down Expand Up @@ -436,6 +440,7 @@ def _prepare_deploy_plan(
return _DeployPlan(
name=resolved_name,
registry_id=registry_id,
placement=placement,
runtime=normalized_runtime,
runtime_config=loaded_runtime_config,
env_vars=env_vars,
Expand All @@ -452,6 +457,7 @@ def deploy_environment(
no_cache: bool = False,
verbose: bool = False,
registry_id: str | None = None,
project: str | None = None,
build_args: list[str] | None = None,
build_secrets: list[str] | None = None,
runtime: str | None = None,
Expand Down Expand Up @@ -490,6 +496,7 @@ def deploy_environment(
env_file=env_file,
no_env=no_env,
registry_id=registry_id,
project=project,
build_args=build_args,
build_secrets=build_secrets,
runtime=runtime,
Expand Down Expand Up @@ -554,6 +561,7 @@ async def _trigger_build(
key: value
for key, value in (
("registry_id", plan.registry_id),
("project_id", plan.placement.project_id),
("runtime_provider", plan.runtime),
(
"runtime_config",
Expand Down Expand Up @@ -631,7 +639,7 @@ async def _deploy_async(

# Save immediately after trigger so rebuilds work even if streaming crashes.
if env_dir and registry_id:
_save_deploy_link(env_dir, registry_id, console, env_name=plan.name)
_save_deploy_link(env_dir, registry_id, console, env_name=plan.name, plan=plan)

console.success(f"Build triggered [{time.time() - step_start:.1f}s]")
console.info(f"Build ID: {build_id}")
Expand Down Expand Up @@ -679,12 +687,15 @@ def _save_deploy_link(
registry_id: str,
console: HUDConsole,
env_name: str | None = None,
plan: _DeployPlan | None = None,
) -> None:
"""Save deploy linking info to .hud/config.json."""
try:
config_data: dict[str, Any] = {"registryId": registry_id}
if env_name:
config_data["registryName"] = env_name
if plan is not None and plan.placement.project is not None:
config_data["projectId"] = plan.placement.project.id
changed = EnvironmentSource.open(env_dir).save_config(config_data)
console.success(f"Linked to environment: {registry_id[:8]}...")
if changed:
Expand Down Expand Up @@ -712,6 +723,7 @@ def deploy_all(
no_env: bool = False,
no_cache: bool = False,
verbose: bool = False,
project: str | None = None,
build_args: list[str] | None = None,
build_secrets: list[str] | None = None,
runtime: str | None = None,
Expand Down Expand Up @@ -752,6 +764,7 @@ def deploy_all(
no_cache=no_cache,
verbose=verbose,
registry_id=None,
project=project,
build_args=build_args,
build_secrets=build_secrets,
runtime=runtime,
Expand Down Expand Up @@ -830,6 +843,12 @@ def deploy_command(
help="Existing registry ID for rebuilds (advanced)",
hidden=True,
),
project: str | None = typer.Option(
None,
"--project",
help="Project to create this environment in (name or ID). Defaults to the "
"directory's saved project, then HUD_PROJECT, then your team default.",
),
runtime: str | None = typer.Option(
None,
"--runtime",
Expand All @@ -856,6 +875,7 @@ def deploy_command(
no_env=no_env,
no_cache=no_cache,
verbose=verbose,
project=project,
build_args=build_args,
build_secrets=secrets,
runtime=runtime,
Expand All @@ -871,6 +891,7 @@ def deploy_command(
no_cache=no_cache,
verbose=verbose,
registry_id=registry_id,
project=project,
build_args=build_args,
build_secrets=secrets,
runtime=runtime,
Expand Down
169 changes: 169 additions & 0 deletions hud/cli/project.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
"""``hud project`` — see and choose the Project new environments land in."""

from __future__ import annotations

import httpx
import typer

from hud.cli.utils.api import require_api_key
from hud.cli.utils.project import (
Project,
ProjectNotFound,
ProjectNotWritable,
list_projects,
report_project_error,
resolve_placement,
resolve_project,
)
from hud.cli.utils.source import EnvironmentSource
from hud.utils.exceptions import HudRequestError
from hud.utils.hud_console import HUDConsole
from hud.utils.platform import PlatformClient

project_app = typer.Typer(
name="project",
help="Show and choose the HUD Project new environments and tasksets land in",
add_completion=False,
rich_markup_mode="rich",
)


@project_app.command("list")
def list_command() -> None:
"""List the Projects you can see.

[not dim]Examples:
hud project list[/not dim]
"""
console = HUDConsole()
require_api_key("list projects")

try:
projects = list_projects(PlatformClient.from_settings())
except HudRequestError as e:
raise report_project_error(console, e) from e

if not projects:
console.warning("No projects found")
console.hint("Create one with: hud project create <name>")
return

console.info("Your projects:")
for project in sorted(projects, key=lambda p: (not p.is_default, p.name)):
tags: list[str] = []
if project.is_default:
tags.append("default")
if not project.can_create:
tags.append("read-only")
suffix = f" [{', '.join(tags)}]" if tags else ""
console.info(f" {project.name} ({project.short_id}...){suffix}")


@project_app.command("create")
def create_command(
name: str = typer.Argument(..., help="Name for the new project"),
description: str | None = typer.Option(None, "--description", help="What the project holds"),
directory: str = typer.Option(".", "--directory", "-C", help="Directory to pin it to"),
no_use: bool = typer.Option(False, "--no-use", help="Create without pinning this directory"),
) -> None:
"""Create a Project and pin this directory to it.

[not dim]Only team admins can create projects.

Examples:
hud project create browser-evals
hud project create browser-evals --no-use[/not dim]
"""
console = HUDConsole()
require_api_key("create a project")

platform = PlatformClient.from_settings()
payload: dict[str, str] = {"name": name}
if description:
payload["description"] = description

try:
created = Project.from_record(platform.post("/projects", json=payload))
except HudRequestError as e:
if e.status_code == httpx.codes.CONFLICT:
console.error(f"A project named '{name}' already exists")
console.hint(f"Pin this directory to it with: hud project use {name}")
raise typer.Exit(1) from e
if e.status_code == httpx.codes.FORBIDDEN:
console.error("Only team admins can create projects")
raise typer.Exit(1) from e
raise report_project_error(console, e) from e

console.success(f"Created project: {created.name} ({created.short_id}...)")
if not no_use:
_pin(created, directory, console)


@project_app.command("use")
def use_command(
ref: str = typer.Argument(..., help="Project name or ID"),
directory: str = typer.Option(".", "--directory", "-C", help="Directory to pin"),
) -> None:
"""Pin a directory to a Project.

[not dim]Writes projectId to .hud/config.json, so teammates deploying this
environment place it in the same project. Set a machine-wide fallback for
unpinned directories with: hud set HUD_PROJECT=<name>

Examples:
hud project use browser-evals
hud project use browser-evals -C ./envs/browser[/not dim]
"""
console = HUDConsole()
require_api_key("select a project")

try:
project = resolve_project(PlatformClient.from_settings(), ref)
except (ProjectNotFound, HudRequestError) as e:
raise report_project_error(console, e) from e
if not project.can_create:
raise report_project_error(console, ProjectNotWritable(project))
_pin(project, directory, console)


@project_app.callback(invoke_without_command=True)
def project_callback(
ctx: typer.Context,
directory: str = typer.Option(".", "--directory", "-C", help="Directory to report on"),
) -> None:
"""Show the Project this directory places new environments and tasksets in.

[not dim]Examples:
hud project # where does a deploy here land?
hud project list # projects you can see
hud project use browser-evals # pin this directory[/not dim]
"""
if ctx.invoked_subcommand is not None:
return

console = HUDConsole()
require_api_key("resolve the current project")
try:
placement = resolve_placement(
PlatformClient.from_settings(),
EnvironmentSource.open(directory),
flag=None,
)
except (ProjectNotFound, HudRequestError) as e:
raise report_project_error(console, e) from e

console.info(f"Project: {placement.label}")
if placement.project is None:
console.hint("Pin a different one with: hud project use <name>")
elif not placement.project.can_create:
console.warning("You do not have create access to this Project")


def _pin(project: Project, directory: str, console: HUDConsole) -> None:
changed = EnvironmentSource.open(directory).save_config({"projectId": project.id})
console.success(f"Using project: {project.name} ({project.short_id}...)")
if changed:
console.dim_info("Config saved to:", ".hud/config.json")


__all__ = ["project_app"]
Loading
Loading