Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -235,13 +235,14 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:

config = get_config()
command, extra_env = self._resolve_command(context)
user_overrides = context.metadata.get("user_overrides") or {}
if self.env_id is not None:
target = self._computer(context)
target.env = self._merged_env(config, target.deployed.sandbox_type, extra_env)
target.env = self._merged_env(config, target.deployed.sandbox_type, extra_env, user_overrides)
exit_code, stdout, stderr, extracted_files = await self._run_in_computer(target, command)
else:
sandbox = await self._container_sandbox(context)
merged_env = self._merged_env(config, sandbox.type, extra_env)
merged_env = self._merged_env(config, sandbox.type, extra_env, user_overrides)
exit_code, stdout, stderr, extracted_files = await self._run_in_container(sandbox, merged_env, command)
timed_out = exit_code in (124, 137)
logger.info(
Expand Down Expand Up @@ -343,19 +344,40 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext:

return context

def _merged_env(self, config, sandbox_type: Optional[str], extra_env: dict[str, str]) -> dict[str, str]:
"""The commands' environment: the model endpoint's key and URL from agent-env's config, every sandbox
provider's container env, ``env_vars`` and this run's ``extra_env``, then the endpoint's headers, its URL as
``sandbox_type``'s containers reach it."""
def _merged_env(
self, config, sandbox_type: Optional[str], extra_env: dict[str, str],
user_overrides: Optional[dict[str, Any]] = None,
) -> dict[str, str]:
"""The commands' environment: the model endpoint's key and URL, every sandbox provider's container env,
``env_vars`` and this run's ``extra_env``, then the endpoint's headers, its URL as ``sandbox_type``'s
containers reach it. The key and URL are the run's own (``user_overrides`` ``litellm_api_key`` /
``litellm_base_url``, as ``deploy_agent`` takes them; ``judge_litellm_api_key`` / ``judge_litellm_base_url``
first, as ``rubrics_verifier``)
when it names them, else agent-env's config: a run
launched on a caller's key grades on that key and bills it, not the platform's. The run's URL is taken only
with the run's own key, so agent-env's key never goes to an endpoint a run chose."""
overrides = user_overrides or {}
merged_env: dict[str, str] = {}
# as rubrics_verifier: a judge key and its URL, when the run names one, then the run's own key and URL
judge_key = overrides.get("judge_litellm_api_key")
run_key = judge_key or overrides.get("litellm_api_key")
run_url = (
(overrides.get("judge_litellm_base_url") or overrides.get("litellm_base_url"))
if judge_key
else overrides.get("litellm_base_url")
)
try:
litellm_key = config.get_litellm_api_key()
litellm_key = config.resolve_reference(run_key) if run_key else config.get_litellm_api_key()
merged_env["LITELLM_API_KEY"] = litellm_key
merged_env["ANTHROPIC_API_KEY"] = litellm_key
except (KeyError, ConfigError):
logger.warning("LITELLM_API_KEY unavailable; verifier won't have LLM creds")
if run_key:
# never agent-env's key instead: the run named its own, and would be billed for none of it
logger.warning("this run's LiteLLM key could not be resolved; verifier won't have LLM creds")
else:
logger.warning("LITELLM_API_KEY unavailable; verifier won't have LLM creds")
try:
litellm_url = config.get_litellm_base_url()
litellm_url = (run_url if run_key else None) or config.get_litellm_base_url()
merged_env["LITELLM_BASE_URL"] = litellm_url
merged_env["ANTHROPIC_BASE_URL"] = litellm_url
except (KeyError, ConfigError):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -453,3 +453,122 @@ def test_a_verifier_for_a_computer_that_cant_clear_the_reward_file_grades_nothin
def test_a_verifier_naming_its_target_wrong_is_refused(fields, problem):
with pytest.raises(ValueError, match=problem):
Step(id="scrape", version=None, command="true", **fields)


class _Config:
"""The two config reads ``_merged_env`` makes for the model endpoint, and no headers."""

def get_litellm_api_key(self):
return "sk-platform"

def get_litellm_base_url(self):
return "https://proxy.example"

def resolve_reference(self, value):
return value.removeprefix("env:RESOLVED_") if isinstance(value, str) else value

def get_model_headers(self, base_url):
return {}


@pytest.fixture
def no_provider_env(monkeypatch):
monkeypatch.setattr(verifier_module, "all_sandbox_container_env", lambda: {})


def test_a_run_launched_on_a_callers_key_grades_on_that_key(no_provider_env):
env = _step()._merged_env(_Config(), "modal", {}, {"litellm_api_key": "sk-caller"})
assert env["LITELLM_API_KEY"] == env["ANTHROPIC_API_KEY"] == "sk-caller"


def test_a_runs_judge_key_wins_over_its_agent_key_as_in_the_rubrics_verifier(no_provider_env):
env = _step()._merged_env(_Config(), "modal", {},
{"litellm_api_key": "sk-agent", "judge_litellm_api_key": "sk-judge"})
assert env["LITELLM_API_KEY"] == env["ANTHROPIC_API_KEY"] == "sk-judge"


def test_a_runs_judge_key_goes_to_the_judge_endpoint_not_the_agents(no_provider_env):
env = _step()._merged_env(_Config(), "modal", {}, {
"litellm_api_key": "sk-agent", "litellm_base_url": "https://agent.example",
"judge_litellm_api_key": "sk-judge", "judge_litellm_base_url": "https://judge.example"})
assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-judge", "https://judge.example")
# as in the rubrics verifier: a judge key with no judge URL goes to the run's URL
env = _step()._merged_env(_Config(), "modal", {}, {
"litellm_api_key": "sk-agent", "litellm_base_url": "https://agent.example", "judge_litellm_api_key": "sk-judge"})
assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-judge", "https://agent.example")


def test_a_judge_url_without_a_judge_key_never_moves_the_runs_key(no_provider_env):
env = _step()._merged_env(_Config(), "modal", {}, {
"litellm_api_key": "sk-agent", "judge_litellm_base_url": "https://judge.example"})
assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-agent", "https://proxy.example")


def test_a_run_key_given_as_a_reference_is_resolved(no_provider_env):
env = _step()._merged_env(_Config(), "modal", {}, {"litellm_api_key": "env:RESOLVED_sk-ref"})
assert env["LITELLM_API_KEY"] == "sk-ref"


def test_a_run_without_its_own_key_or_url_keeps_the_platforms(no_provider_env):
env = _step()._merged_env(_Config(), "modal", {}, {"step_params": {}})
assert env["LITELLM_API_KEY"] == "sk-platform"
assert env["LITELLM_BASE_URL"] == env["ANTHROPIC_BASE_URL"] == "https://proxy.example"


def test_a_run_endpoint_override_moves_the_url_with_its_own_key(no_provider_env):
env = _step()._merged_env(_Config(), "modal", {},
{"litellm_api_key": "sk-caller", "litellm_base_url": "https://other.example"})
assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-caller", "https://other.example")


def test_the_platforms_key_never_goes_to_a_url_the_run_chose(no_provider_env):
env = _step()._merged_env(_Config(), "modal", {}, {"litellm_base_url": "https://other.example"})
assert (env["LITELLM_API_KEY"], env["LITELLM_BASE_URL"]) == ("sk-platform", "https://proxy.example")


def test_a_run_key_that_cannot_be_resolved_is_said_and_never_swapped_for_the_platforms(no_provider_env, caplog):
class _Unresolvable(_Config):
def resolve_reference(self, value):
raise KeyError(value)

with caplog.at_level("WARNING"):
env = _step()._merged_env(_Unresolvable(), "modal", {}, {"litellm_api_key": "env:MISSING"})
assert "LITELLM_API_KEY" not in env and "ANTHROPIC_API_KEY" not in env
assert "this run's LiteLLM key could not be resolved" in caplog.text and "MISSING" not in caplog.text


def test_a_key_the_step_itself_sets_still_wins_over_the_runs(no_provider_env):
step = _step(env_vars={"LITELLM_API_KEY": "sk-step"})
env = step._merged_env(_Config(), "modal", {}, {"litellm_api_key": "sk-caller"})
assert env["LITELLM_API_KEY"] == "sk-step"


def test_the_executed_verifier_hands_the_runs_key_to_its_container(local_stores, monkeypatch):
"""Through ``execute``: the hub's worker puts the launcher's decrypted key in ``user_overrides``."""
monkeypatch.setenv("LITELLM_API_KEY", "sk-platform")
configure()
commands: list[str] = []

class _Recording(_Sandbox):
async def exec_script(self, script: str) -> str:
commands.append(script)
return ""

async def exec_with_output(self, *args) -> tuple[int, str, str]:
commands.extend(a for a in args if isinstance(a, str))
return 0, "ok", ""

provider = SimpleNamespace(get_sandbox=lambda sandbox_id: asyncio.sleep(0, result=_Recording()))
monkeypatch.setattr(sandbox_provider, "get_sandbox_provider", lambda: provider)
monkeypatch.setattr(Step, "_upload_text_artifact", staticmethod(
lambda text, artifact_id, description, object_url: SimpleNamespace(
id=artifact_id, version=1, object_url=object_url)))
ctx = TaskStepContext()
ctx.metadata["deployed_docker_containers"] = [{"container_name": "c", "sandbox_name": "h"}]
ctx.deployed_sandboxes.append(DeployedSandbox(sandbox_name="h", sandbox_id="sb-1", sandbox_mode="vm"))
ctx.metadata["user_overrides"] = {"litellm_api_key": "sk-caller"}

asyncio.run(_step(command="true").execute(ctx))

sent = " ".join(commands)
assert "LITELLM_API_KEY=sk-caller" in sent and "sk-platform" not in sent
Loading