From 46c5f0962649aeba92da6e2d134aa4ede307a327 Mon Sep 17 00:00:00 2001 From: Asadullo Ganiev <62354884+solvemproblr@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:33:28 +0500 Subject: [PATCH] feat(cli): add project placement --- README.md | 10 ++ hud/cli/__init__.py | 2 + hud/cli/deploy.py | 23 +++- hud/cli/project.py | 169 ++++++++++++++++++++++++ hud/cli/sync.py | 29 ++++- hud/cli/tests/test_deploy.py | 92 +++++++++++++ hud/cli/utils/project.py | 194 ++++++++++++++++++++++++++++ hud/cli/utils/source.py | 6 + hud/cli/utils/tests/test_project.py | 178 +++++++++++++++++++++++++ hud/eval/sync.py | 4 + hud/eval/tests/test_sync.py | 35 +++++ hud/settings.py | 7 + 12 files changed, 744 insertions(+), 5 deletions(-) create mode 100644 hud/cli/project.py create mode 100644 hud/cli/utils/project.py create mode 100644 hud/cli/utils/tests/test_project.py diff --git a/README.md b/README.md index 4a06edc15..6f75a4d9e 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,16 @@ A **capability** is a connection the environment exposes; a **harness** attaches From the [platform UI](https://hud.ai) you can run batches, compare models on the same taskset, and inspect every trace. +A **project** holds the environments and tasksets a team creates and decides who can see them. Deploying without one uses your team's default project, so nothing here is required to get started. To put an environment somewhere else, pin the directory once and every later `hud deploy` and `hud sync tasks` follows it: + +```bash +hud project list # projects you can use +hud project use browser-evals # writes projectId to .hud/config.json +hud project # where does a deploy here land? +``` + +Both commands also take `--project ` to override the resolved Project. A successful deploy or sync pins that Project to the directory; `hud set HUD_PROJECT=` instead sets a machine-wide default for directories you have not pinned. Precedence is the flag, then the directory's `.hud/config.json`, then `HUD_PROJECT`, then your team default. An environment or taskset that already exists stays where it is; naming a different project fails rather than moving it. + → [Run & deploy](https://docs.hud.ai/v6/reference/runtime) ## Train on rewards diff --git a/hud/cli/__init__.py b/hud/cli/__init__.py index a228ae4d9..c8336d29b 100644 --- a/hud/cli/__init__.py +++ b/hud/cli/__init__.py @@ -38,6 +38,7 @@ from .jobs import jobs_app # noqa: E402 from .login import login_command # noqa: E402 from .models import models_app # noqa: E402 +from .project import project_app # noqa: E402 from .qa import qa_app # noqa: E402 from .serve import serve_command # noqa: E402 from .sync import sync_app # noqa: E402 @@ -54,6 +55,7 @@ app.add_typer(jobs_app, name="jobs") app.add_typer(trace_app, name="trace") app.add_typer(qa_app, name="qa") +app.add_typer(project_app, name="project") @app.command(name="set") diff --git a/hud/cli/deploy.py b/hud/cli/deploy.py index 815d6e42e..9de8788f0 100644 --- a/hud/cli/deploy.py +++ b/hud/cli/deploy.py @@ -19,6 +19,7 @@ from hud.cli.utils.build_logs import poll_build_status, stream_build_logs from hud.cli.utils.config import parse_env_file, parse_key_value from hud.cli.utils.context import create_build_context_tarball, format_size +from hud.cli.utils.project import Placement, resolve_writable_placement from hud.cli.utils.registry import get_registry_environment from hud.cli.utils.source import EnvironmentSource from hud.eval.runtime import RuntimeConfig @@ -41,6 +42,7 @@ class _DeployPlan: name: str registry_id: str | None + placement: Placement runtime: str | None runtime_config: RuntimeConfig | None env_vars: dict[str, str] @@ -371,6 +373,7 @@ def _prepare_deploy_plan( env_file: str | None, no_env: bool, registry_id: str | None, + project: str | None, build_args: list[str] | None, build_secrets: list[str] | None, runtime: str | None, @@ -386,6 +389,7 @@ def _prepare_deploy_plan( platform, console, ) + placement = resolve_writable_placement(platform, env_source, flag=project, console=console) skip_dotenv = _skip_dotenv( env_source, env_dir, @@ -436,6 +440,7 @@ def _prepare_deploy_plan( return _DeployPlan( name=resolved_name, registry_id=registry_id, + placement=placement, runtime=normalized_runtime, runtime_config=loaded_runtime_config, env_vars=env_vars, @@ -452,6 +457,7 @@ def deploy_environment( no_cache: bool = False, verbose: bool = False, registry_id: str | None = None, + project: str | None = None, build_args: list[str] | None = None, build_secrets: list[str] | None = None, runtime: str | None = None, @@ -490,6 +496,7 @@ def deploy_environment( env_file=env_file, no_env=no_env, registry_id=registry_id, + project=project, build_args=build_args, build_secrets=build_secrets, runtime=runtime, @@ -554,6 +561,7 @@ async def _trigger_build( key: value for key, value in ( ("registry_id", plan.registry_id), + ("project_id", plan.placement.project_id), ("runtime_provider", plan.runtime), ( "runtime_config", @@ -631,7 +639,7 @@ async def _deploy_async( # Save immediately after trigger so rebuilds work even if streaming crashes. if env_dir and registry_id: - _save_deploy_link(env_dir, registry_id, console, env_name=plan.name) + _save_deploy_link(env_dir, registry_id, console, env_name=plan.name, plan=plan) console.success(f"Build triggered [{time.time() - step_start:.1f}s]") console.info(f"Build ID: {build_id}") @@ -679,12 +687,15 @@ def _save_deploy_link( registry_id: str, console: HUDConsole, env_name: str | None = None, + plan: _DeployPlan | None = None, ) -> None: """Save deploy linking info to .hud/config.json.""" try: config_data: dict[str, Any] = {"registryId": registry_id} if env_name: config_data["registryName"] = env_name + if plan is not None and plan.placement.project is not None: + config_data["projectId"] = plan.placement.project.id changed = EnvironmentSource.open(env_dir).save_config(config_data) console.success(f"Linked to environment: {registry_id[:8]}...") if changed: @@ -712,6 +723,7 @@ def deploy_all( no_env: bool = False, no_cache: bool = False, verbose: bool = False, + project: str | None = None, build_args: list[str] | None = None, build_secrets: list[str] | None = None, runtime: str | None = None, @@ -752,6 +764,7 @@ def deploy_all( no_cache=no_cache, verbose=verbose, registry_id=None, + project=project, build_args=build_args, build_secrets=build_secrets, runtime=runtime, @@ -830,6 +843,12 @@ def deploy_command( help="Existing registry ID for rebuilds (advanced)", hidden=True, ), + project: str | None = typer.Option( + None, + "--project", + help="Project to create this environment in (name or ID). Defaults to the " + "directory's saved project, then HUD_PROJECT, then your team default.", + ), runtime: str | None = typer.Option( None, "--runtime", @@ -856,6 +875,7 @@ def deploy_command( no_env=no_env, no_cache=no_cache, verbose=verbose, + project=project, build_args=build_args, build_secrets=secrets, runtime=runtime, @@ -871,6 +891,7 @@ def deploy_command( no_cache=no_cache, verbose=verbose, registry_id=registry_id, + project=project, build_args=build_args, build_secrets=secrets, runtime=runtime, diff --git a/hud/cli/project.py b/hud/cli/project.py new file mode 100644 index 000000000..3189d9ffb --- /dev/null +++ b/hud/cli/project.py @@ -0,0 +1,169 @@ +"""``hud project`` — see and choose the Project new environments land in.""" + +from __future__ import annotations + +import httpx +import typer + +from hud.cli.utils.api import require_api_key +from hud.cli.utils.project import ( + Project, + ProjectNotFound, + ProjectNotWritable, + list_projects, + report_project_error, + resolve_placement, + resolve_project, +) +from hud.cli.utils.source import EnvironmentSource +from hud.utils.exceptions import HudRequestError +from hud.utils.hud_console import HUDConsole +from hud.utils.platform import PlatformClient + +project_app = typer.Typer( + name="project", + help="Show and choose the HUD Project new environments and tasksets land in", + add_completion=False, + rich_markup_mode="rich", +) + + +@project_app.command("list") +def list_command() -> None: + """List the Projects you can see. + + [not dim]Examples: + hud project list[/not dim] + """ + console = HUDConsole() + require_api_key("list projects") + + try: + projects = list_projects(PlatformClient.from_settings()) + except HudRequestError as e: + raise report_project_error(console, e) from e + + if not projects: + console.warning("No projects found") + console.hint("Create one with: hud project create ") + return + + console.info("Your projects:") + for project in sorted(projects, key=lambda p: (not p.is_default, p.name)): + tags: list[str] = [] + if project.is_default: + tags.append("default") + if not project.can_create: + tags.append("read-only") + suffix = f" [{', '.join(tags)}]" if tags else "" + console.info(f" {project.name} ({project.short_id}...){suffix}") + + +@project_app.command("create") +def create_command( + name: str = typer.Argument(..., help="Name for the new project"), + description: str | None = typer.Option(None, "--description", help="What the project holds"), + directory: str = typer.Option(".", "--directory", "-C", help="Directory to pin it to"), + no_use: bool = typer.Option(False, "--no-use", help="Create without pinning this directory"), +) -> None: + """Create a Project and pin this directory to it. + + [not dim]Only team admins can create projects. + + Examples: + hud project create browser-evals + hud project create browser-evals --no-use[/not dim] + """ + console = HUDConsole() + require_api_key("create a project") + + platform = PlatformClient.from_settings() + payload: dict[str, str] = {"name": name} + if description: + payload["description"] = description + + try: + created = Project.from_record(platform.post("/projects", json=payload)) + except HudRequestError as e: + if e.status_code == httpx.codes.CONFLICT: + console.error(f"A project named '{name}' already exists") + console.hint(f"Pin this directory to it with: hud project use {name}") + raise typer.Exit(1) from e + if e.status_code == httpx.codes.FORBIDDEN: + console.error("Only team admins can create projects") + raise typer.Exit(1) from e + raise report_project_error(console, e) from e + + console.success(f"Created project: {created.name} ({created.short_id}...)") + if not no_use: + _pin(created, directory, console) + + +@project_app.command("use") +def use_command( + ref: str = typer.Argument(..., help="Project name or ID"), + directory: str = typer.Option(".", "--directory", "-C", help="Directory to pin"), +) -> None: + """Pin a directory to a Project. + + [not dim]Writes projectId to .hud/config.json, so teammates deploying this + environment place it in the same project. Set a machine-wide fallback for + unpinned directories with: hud set HUD_PROJECT= + + Examples: + hud project use browser-evals + hud project use browser-evals -C ./envs/browser[/not dim] + """ + console = HUDConsole() + require_api_key("select a project") + + try: + project = resolve_project(PlatformClient.from_settings(), ref) + except (ProjectNotFound, HudRequestError) as e: + raise report_project_error(console, e) from e + if not project.can_create: + raise report_project_error(console, ProjectNotWritable(project)) + _pin(project, directory, console) + + +@project_app.callback(invoke_without_command=True) +def project_callback( + ctx: typer.Context, + directory: str = typer.Option(".", "--directory", "-C", help="Directory to report on"), +) -> None: + """Show the Project this directory places new environments and tasksets in. + + [not dim]Examples: + hud project # where does a deploy here land? + hud project list # projects you can see + hud project use browser-evals # pin this directory[/not dim] + """ + if ctx.invoked_subcommand is not None: + return + + console = HUDConsole() + require_api_key("resolve the current project") + try: + placement = resolve_placement( + PlatformClient.from_settings(), + EnvironmentSource.open(directory), + flag=None, + ) + except (ProjectNotFound, HudRequestError) as e: + raise report_project_error(console, e) from e + + console.info(f"Project: {placement.label}") + if placement.project is None: + console.hint("Pin a different one with: hud project use ") + elif not placement.project.can_create: + console.warning("You do not have create access to this Project") + + +def _pin(project: Project, directory: str, console: HUDConsole) -> None: + changed = EnvironmentSource.open(directory).save_config({"projectId": project.id}) + console.success(f"Using project: {project.name} ({project.short_id}...)") + if changed: + console.dim_info("Config saved to:", ".hud/config.json") + + +__all__ = ["project_app"] diff --git a/hud/cli/sync.py b/hud/cli/sync.py index c0aaf9275..ab430eea8 100644 --- a/hud/cli/sync.py +++ b/hud/cli/sync.py @@ -11,6 +11,7 @@ import typer from hud.cli.utils.api import require_api_key +from hud.cli.utils.project import Placement, resolve_writable_placement from hud.cli.utils.registry import ( RegistryEnvironment, get_registry_environment, @@ -217,11 +218,14 @@ def _show_upload_error(error: HudRequestError, console: HUDConsole) -> None: console.error(f"Upload failed ({error.status_code}): {detail or error}") -def _save_taskset_id(result: dict[str, object], console: HUDConsole) -> None: +def _save_taskset_id(result: dict[str, object], placement: Placement, console: HUDConsole) -> None: returned_id = result.get("taskset_id") if not isinstance(returned_id, str) or not returned_id: return - changed = EnvironmentSource.open().save_config({"tasksetId": returned_id}) + config: dict[str, object] = {"tasksetId": returned_id} + if placement.project is not None: + config["projectId"] = placement.project.id + changed = EnvironmentSource.open().save_config(config) if changed: console.dim_info("Taskset ID saved to:", ".hud/config.json") from hud.settings import settings @@ -244,6 +248,12 @@ def sync_tasks_command( "--id", help="Taskset ID directly (skip name resolution)", ), + project: str | None = typer.Option( + None, + "--project", + help="Project to create this taskset in (name or ID). Defaults to the " + "directory's saved project, then HUD_PROJECT, then your team default.", + ), task_filter: str | None = typer.Option( None, "--task", @@ -315,6 +325,12 @@ def sync_tasks_command( # Creating a new taskset is only allowed when targeting an explicit name # (not an --id or a stored id, which must already exist). allow_create = taskset is not None and taskset_id is None + placement = resolve_writable_placement( + platform, + EnvironmentSource.open(), + flag=project, + console=hud_console, + ) try: remote_taskset = _fetch_remote_taskset( @@ -352,7 +368,12 @@ def sync_tasks_command( # Upload tasks; the platform validates referenced environments. hud_console.progress_message("Uploading tasks...") try: - result = upload_taskset(platform, plan.taskset_name, plan.to_apply) + result = upload_taskset( + platform, + plan.taskset_name, + plan.to_apply, + project_id=placement.project_id, + ) except HudRequestError as e: _show_upload_error(e, hud_console) return @@ -362,7 +383,7 @@ def sync_tasks_command( hud_console.success("Sync complete") hud_console.info(f" + {created} created, ~ {updated} updated") - _save_taskset_id(result, hud_console) + _save_taskset_id(result, placement, hud_console) @sync_app.command("env") diff --git a/hud/cli/tests/test_deploy.py b/hud/cli/tests/test_deploy.py index 1533bdc08..8dd73e390 100644 --- a/hud/cli/tests/test_deploy.py +++ b/hud/cli/tests/test_deploy.py @@ -10,11 +10,15 @@ import typer from hud.cli.deploy import _resolve_environment_name +from hud.cli.utils.project import Placement, Project, ProjectSource from hud.cli.utils.registry import RegistryEnvironment from hud.cli.utils.source import EnvironmentSource from hud.utils.hud_console import HUDConsole from hud.utils.platform import PlatformClient +# Deploys that accept the team's default Project send no project_id. +_UNPLACED = Placement(project=None, source=ProjectSource.TEAM_DEFAULT) + @pytest.mark.parametrize(("value", "expected"), [("HUD", "hud"), ("modal", "modal")]) def test_normalize_runtime_uses_public_runtime_names(value: str, expected: str) -> None: @@ -232,6 +236,7 @@ def test_prepare_deploy_uses_context_recipe( env_file=None, no_env=True, registry_id=None, + project=None, build_args=None, build_secrets=None, runtime=None, @@ -331,6 +336,7 @@ def test_prepare_deploy_rejects_image_config_for_compose_context( env_file=None, no_env=True, registry_id=None, + project=None, build_args=None, build_secrets=None, runtime=None, @@ -450,6 +456,7 @@ async def test_upload_url_failure(self) -> None: plan=_DeployPlan( name="test-env", registry_id=None, + placement=_UNPLACED, runtime=None, runtime_config=None, env_vars={}, @@ -481,6 +488,7 @@ async def test_upload_url_network_error(self) -> None: plan=_DeployPlan( name="test-env", registry_id=None, + placement=_UNPLACED, runtime=None, runtime_config=None, env_vars={}, @@ -493,6 +501,90 @@ async def test_upload_url_network_error(self) -> None: assert result.success is False + @pytest.mark.asyncio + async def test_trigger_build_sends_resolved_project(self) -> None: + """A resolved placement reaches the platform as project_id.""" + from hud.cli.deploy import _DeployPlan, _trigger_build + from hud.utils.platform import PlatformClient + + class FakePlatform(PlatformClient): + payload: dict[str, object] | None = None + + async def apost( + self, + path: str, + *, + json: object | None = None, + ) -> dict[str, object]: + object.__setattr__(self, "payload", json) + return {"id": "build-1", "registry_id": "registry-1"} + + platform = FakePlatform("https://api.example", "key") + await _trigger_build( + platform, + build_id="build-1", + plan=_DeployPlan( + name="test-env", + registry_id=None, + placement=Placement( + project=Project( + id="project-1", + name="browser-evals", + is_default=False, + can_create=True, + ), + source=ProjectSource.FLAG, + ), + runtime=None, + runtime_config=None, + env_vars={}, + build_args={}, + build_secrets={}, + ), + no_cache=False, + ) + + assert platform.payload is not None + assert platform.payload["project_id"] == "project-1" + + @pytest.mark.asyncio + async def test_trigger_build_omits_project_for_the_team_default(self) -> None: + """The zero-config deploy stays byte-identical to before projects existed.""" + from hud.cli.deploy import _DeployPlan, _trigger_build + from hud.utils.platform import PlatformClient + + class FakePlatform(PlatformClient): + payload: dict[str, object] | None = None + + async def apost( + self, + path: str, + *, + json: object | None = None, + ) -> dict[str, object]: + object.__setattr__(self, "payload", json) + return {"id": "build-1", "registry_id": "registry-1"} + + platform = FakePlatform("https://api.example", "key") + await _trigger_build( + platform, + build_id="build-1", + plan=_DeployPlan( + name="test-env", + registry_id=None, + placement=_UNPLACED, + runtime=None, + runtime_config=None, + env_vars={}, + build_args={}, + build_secrets={}, + ), + no_cache=False, + ) + + assert platform.payload is not None + assert "project_id" not in platform.payload + class TestSaveDeployLink: """Tests for _save_deploy_link function.""" diff --git a/hud/cli/utils/project.py b/hud/cli/utils/project.py new file mode 100644 index 000000000..b2484b63a --- /dev/null +++ b/hud/cli/utils/project.py @@ -0,0 +1,194 @@ +"""Project lookup and placement resolution for the CLI.""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING, Any + +import typer + +from hud.utils.naming import normalize_environment_name + +if TYPE_CHECKING: + from hud.cli.utils.source import EnvironmentSource + from hud.utils.hud_console import HUDConsole + from hud.utils.platform import PlatformClient + + +class ProjectSource(Enum): + """Where a resolved Project came from, most specific first.""" + + FLAG = "--project" + CONFIG = ".hud/config.json" + SETTINGS = "HUD_PROJECT" + TEAM_DEFAULT = "team default" + + +@dataclass(frozen=True) +class Project: + id: str + name: str + is_default: bool + can_create: bool + + @classmethod + def from_record(cls, data: dict[str, Any]) -> Project: + capabilities = data.get("capabilities") + return cls( + id=str(data["id"]), + name=str(data.get("name") or "unnamed"), + is_default=bool(data.get("is_default")), + can_create=bool(capabilities.get("create")) + if isinstance(capabilities, dict) + else False, + ) + + @property + def short_id(self) -> str: + return self.id[:8] + + +@dataclass(frozen=True) +class Placement: + """The Project selected for the current directory.""" + + project: Project | None + source: ProjectSource + + @property + def project_id(self) -> str | None: + """The id to send to the platform, or None to accept the team default.""" + return self.project.id if self.project else None + + @property + def label(self) -> str: + if self.project is None: + return "team default Project" + return f"{self.project.name} (via {self.source.value})" + + +class ProjectNotFound(LookupError): + """No visible Project matches the given reference.""" + + def __init__(self, ref: str, available: list[Project]) -> None: + self.ref = ref + self.available = available + super().__init__(f"No project found matching '{ref}'") + + +class ProjectNotWritable(PermissionError): + """The caller may see the Project but may not create resources in it.""" + + def __init__(self, project: Project) -> None: + self.project = project + super().__init__( + f"You do not have permission to create environments or tasksets in " + f"project '{project.name}'" + ) + + +def list_projects(platform: PlatformClient) -> list[Project]: + """Every Project visible to the caller.""" + data = platform.get("/projects") + records = data.get("projects") if isinstance(data, dict) else None + if not isinstance(records, list): + return [] + return [Project.from_record(item) for item in records if isinstance(item, dict)] + + +def resolve_project(platform: PlatformClient, ref: str) -> Project: + """Map a Project name or id to the Project itself. + + Names are normalized the same way the platform normalizes them on create, + so `My Project` and `my-project` resolve to the same row. + """ + projects = list_projects(platform) + try: + project_id = str(uuid.UUID(ref)) + except ValueError: + project_id = None + + match = next((p for p in projects if p.id == project_id), None) + if match is None: + name = normalize_environment_name(ref, default="") + match = next((p for p in projects if p.name == name), None) + + if match is None: + raise ProjectNotFound(ref, projects) + return match + + +def resolve_placement( + platform: PlatformClient, + env_source: EnvironmentSource, + *, + flag: str | None, +) -> Placement: + """Resolve the configured Project.""" + from hud.settings import settings + + for ref, source in ( + (flag, ProjectSource.FLAG), + (env_source.project_id, ProjectSource.CONFIG), + (settings.project, ProjectSource.SETTINGS), + ): + if ref: + project = resolve_project(platform, ref) + return Placement(project=project, source=source) + + return Placement(project=None, source=ProjectSource.TEAM_DEFAULT) + + +def report_project_error(console: HUDConsole, error: Exception) -> typer.Exit: + """Explain why a Project could not be used, and return the exit to raise.""" + if isinstance(error, ProjectNotFound): + console.error(str(error)) + if error.available: + console.info("Projects you can see:") + for candidate in error.available: + console.info(f" {candidate.name} ({candidate.short_id}...)") + else: + console.hint("Create one with: hud project create ") + elif isinstance(error, ProjectNotWritable): + console.error(str(error)) + console.hint("Ask a project manager for 'create' scope, or pick another project") + else: + console.error(f"Failed to reach the HUD platform: {error}") + return typer.Exit(1) + + +def resolve_writable_placement( + platform: PlatformClient, + env_source: EnvironmentSource, + *, + flag: str | None, + console: HUDConsole, +) -> Placement: + """Resolve and announce a Project that accepts new resources.""" + from hud.utils.exceptions import HudRequestError + + try: + placement = resolve_placement(platform, env_source, flag=flag) + if placement.project is not None and not placement.project.can_create: + raise ProjectNotWritable(placement.project) + except (ProjectNotFound, ProjectNotWritable, HudRequestError) as e: + raise report_project_error(console, e) from e + + console.info(f"Project: {placement.label}") + return placement + + +__all__ = [ + "Placement", + "Project", + "ProjectNotFound", + "ProjectNotWritable", + "ProjectSource", + "list_projects", + "report_project_error", + "resolve_placement", + "resolve_project", + "resolve_writable_placement", +] diff --git a/hud/cli/utils/source.py b/hud/cli/utils/source.py index 33fcbaac3..a7638828a 100644 --- a/hud/cli/utils/source.py +++ b/hud/cli/utils/source.py @@ -210,6 +210,12 @@ def taskset_id(self) -> str | None: value = self.load_config().get("tasksetId") return value if isinstance(value, str) else None + @property + def project_id(self) -> str | None: + """The Project this directory's environment and taskset belong to.""" + value = self.load_config().get("projectId") + return value if isinstance(value, str) else None + def iter_source_files(self) -> Iterator[Path]: for name in self.SOURCE_INCLUDE_FILES: path = self.root / name diff --git a/hud/cli/utils/tests/test_project.py b/hud/cli/utils/tests/test_project.py new file mode 100644 index 000000000..2eb6cc1bd --- /dev/null +++ b/hud/cli/utils/tests/test_project.py @@ -0,0 +1,178 @@ +"""Project lookup and placement precedence for CLI create-and-link flows.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest +import typer + +from hud.cli.utils.project import ( + Project, + ProjectNotFound, + ProjectSource, + resolve_placement, + resolve_project, + resolve_writable_placement, +) +from hud.cli.utils.source import EnvironmentSource +from hud.utils.hud_console import HUDConsole +from hud.utils.platform import PlatformClient + +if TYPE_CHECKING: + from pathlib import Path + +_DEFAULT_ID = "11111111-1111-4111-8111-111111111111" +_BROWSER_ID = "22222222-2222-4222-8222-222222222222" +_READONLY_ID = "33333333-3333-4333-8333-333333333333" + + +def _record( + project_id: str, name: str, *, is_default: bool = False, create: bool = True +) -> dict[str, Any]: + return { + "id": project_id, + "name": name, + "is_default": is_default, + "capabilities": {"view": True, "create": create, "manage": False}, + } + + +@pytest.fixture +def calls() -> list[str]: + """URLs the fake platform transport was asked for.""" + return [] + + +@pytest.fixture +def platform(monkeypatch: pytest.MonkeyPatch, calls: list[str]) -> PlatformClient: + """A client whose ``GET /projects`` returns a fixed three-project team.""" + + def fake_request(method: str, url: str, **kwargs: Any) -> dict[str, Any]: + calls.append(url) + return { + "projects": [ + _record(_DEFAULT_ID, "default", is_default=True), + _record(_BROWSER_ID, "browser-evals"), + _record(_READONLY_ID, "locked-down", create=False), + ] + } + + monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) + return PlatformClient("https://api.example", "key") + + +def _no_settings_project(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("hud.settings.settings.project", None) + + +def test_resolve_matches_a_normalized_name(platform: PlatformClient) -> None: + """A human-typed name resolves through the same normalization the platform applies.""" + assert resolve_project(platform, "Browser Evals").id == _BROWSER_ID + assert resolve_project(platform, "browser-evals").id == _BROWSER_ID + + +def test_resolve_matches_an_id(platform: PlatformClient) -> None: + assert resolve_project(platform, _BROWSER_ID).name == "browser-evals" + assert resolve_project(platform, _BROWSER_ID.upper()).name == "browser-evals" + + +def test_resolve_reports_the_visible_alternatives(platform: PlatformClient) -> None: + with pytest.raises(ProjectNotFound) as excinfo: + resolve_project(platform, "nope") + + assert [p.name for p in excinfo.value.available] == [ + "default", + "browser-evals", + "locked-down", + ] + + +def test_flag_outranks_directory_config( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _no_settings_project(monkeypatch) + source = EnvironmentSource.open(tmp_path) + source.save_config({"projectId": _DEFAULT_ID}) + + placement = resolve_placement(platform, source, flag="browser-evals") + + assert placement.project is not None + assert placement.project.id == _BROWSER_ID + assert placement.source is ProjectSource.FLAG + + +def test_directory_config_outranks_the_machine_default( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Placement is a property of the environment, not of who deploys it.""" + monkeypatch.setattr("hud.settings.settings.project", "default") + source = EnvironmentSource.open(tmp_path) + source.save_config({"projectId": _BROWSER_ID}) + + placement = resolve_placement(platform, source, flag=None) + + assert placement.project is not None + assert placement.project.id == _BROWSER_ID + assert placement.source is ProjectSource.CONFIG + + +def test_machine_default_applies_to_an_unpinned_directory( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("hud.settings.settings.project", "browser-evals") + + placement = resolve_placement(platform, EnvironmentSource.open(tmp_path), flag=None) + + assert placement.project is not None + assert placement.project.id == _BROWSER_ID + assert placement.source is ProjectSource.SETTINGS + + +def test_unconfigured_placement_sends_no_project_and_makes_no_call( + platform: PlatformClient, + calls: list[str], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The zero-config path stays free: no project on the wire, no lookup.""" + _no_settings_project(monkeypatch) + + placement = resolve_placement(platform, EnvironmentSource.open(tmp_path), flag=None) + + assert placement.project_id is None + assert placement.source is ProjectSource.TEAM_DEFAULT + assert placement.label == "team default Project" + assert calls == [] + + +def test_placement_resolves_a_project_the_caller_cannot_create_in( + platform: PlatformClient, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _no_settings_project(monkeypatch) + + source = EnvironmentSource.open(tmp_path) + placement = resolve_placement(platform, source, flag="locked-down") + assert placement.project is not None + assert placement.project.id == _READONLY_ID + + with pytest.raises(typer.Exit): + resolve_writable_placement( + platform, + source, + flag="locked-down", + console=HUDConsole(), + ) + + +def test_from_record_defaults_capabilities_to_read_only() -> None: + """A response without capabilities is not assumed writable.""" + assert Project.from_record({"id": "x", "name": "y"}).can_create is False diff --git a/hud/eval/sync.py b/hud/eval/sync.py index aa37e433a..77e0f7dbd 100644 --- a/hud/eval/sync.py +++ b/hud/eval/sync.py @@ -135,12 +135,16 @@ def upload_taskset( platform: PlatformClient, name: str, tasks: list[Task], + *, + project_id: str | None = None, ) -> dict[str, Any]: """Upload tasks to a platform taskset, creating it if needed.""" payload: dict[str, Any] = { "taskset_name": name, "tasks": [task_upload_payload(task) for task in tasks], } + if project_id: + payload["project_id"] = project_id data = platform.post("/tasks/upload", json=payload) return data if isinstance(data, dict) else {} diff --git a/hud/eval/tests/test_sync.py b/hud/eval/tests/test_sync.py index 71c89eb48..572922b6f 100644 --- a/hud/eval/tests/test_sync.py +++ b/hud/eval/tests/test_sync.py @@ -135,6 +135,41 @@ def fake_request( } +def test_upload_taskset_places_a_new_taskset(monkeypatch: pytest.MonkeyPatch) -> None: + """`project_id` reaches the platform so a created taskset lands in that Project.""" + posted: dict[str, Any] = {} + + def fake_request(method: str, url: str, json: object = None, **kwargs: Any) -> dict[str, Any]: + posted.update(json=json) + return {} + + monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) + + upload_taskset( + PlatformClient("https://api.example", "token"), + "demo", + [], + project_id="project-1", + ) + + assert posted["json"]["project_id"] == "project-1" + + +def test_upload_taskset_omits_project_when_unset(monkeypatch: pytest.MonkeyPatch) -> None: + """Without a project the payload is unchanged, so the default Project applies.""" + posted: dict[str, Any] = {} + + def fake_request(method: str, url: str, json: object = None, **kwargs: Any) -> dict[str, Any]: + posted.update(json=json) + return {} + + monkeypatch.setattr("hud.utils.platform.make_request_sync", fake_request) + + upload_taskset(PlatformClient("https://api.example", "token"), "demo", []) + + assert "project_id" not in posted["json"] + + def test_task_upload_payload_sends_env_and_bare_task_id() -> None: payload = task_upload_payload(Task(env="e", id="solve", args={"n": 1})) diff --git a/hud/settings.py b/hud/settings.py index 5eb64b6eb..d45952a5a 100644 --- a/hud/settings.py +++ b/hud/settings.py @@ -92,6 +92,13 @@ def settings_customise_sources( validation_alias="HUD_API_KEY", ) + project: str | None = Field( + default=None, + description="Default HUD Project (name or id) for environments and tasksets this " + "machine creates. A directory's .hud/config.json takes precedence.", + validation_alias="HUD_PROJECT", + ) + anthropic_api_key: str | None = Field( default=None, description="API key for Anthropic models",